fix(test): Fix licenseRef.json

Replace common unicode with ASCII. Signed-off-by: Gaurav Mishra <[email protected]>
ag4ums · Oct 29, 2021 · 6c241d2 · 6c241d2
1 parent 31dd1c4
commit 6c241d2
Show file tree

Hide file tree

Showing 9 changed files with 211 additions and 165 deletions.
diff --git a/.github/workflows/test-checks.yml b/.github/workflows/test-checks.yml
@@ -1,5 +1,6 @@
-# Copyright 2021 Anupam Ghosh <[email protected]>
-# SPDX-License-Identifier: GPL-2.0 AND LGPL-2.
+# Copyright 2021 Anupam Ghosh <[email protected]>,
+#   Gaurav Mishra <[email protected]>
+# SPDX-License-Identifier: GPL-2.0 AND LGPL-2.1
 
 name: c tests
 
@@ -12,13 +13,16 @@ on:
 jobs:
   C-tests:
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        php: ['7.4'] # Can be extended in future
     services:
       postgres:
         image: postgres:12
         env:
-          POSTGRES_DB: fossology
-          POSTGRES_PASSWORD:  fossy
-          POSTGRES_USER: fossy
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
         ports:
           - 5432:5432
         # Set health checks to wait until postgres has started
@@ -32,21 +36,35 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 50
-      
+
       - name: Install Dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y libcppunit-dev
           sudo ./utils/fo-installdeps -y -e
           export PATH="/usr/lib/ccache/:$PATH"
 
+      - name: Install PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          extensions: mbstring, gd, xml, zip, json, sqlite3, curl
+        env:
+          fail-fast: true
+
       - name: Setup test database
         env:
-          DATABASE_HOST: 127.0.0.1
-          DATABASE_PORT: 5432
+          PGHOST: 127.0.0.1
+          PGPORT: 5432
         run: |
-          sudo ./utils/prepare-test
+          ./utils/prepare-test -afty
 
-          #remove test all as delagent is failing  
+          #remove test all as delagent is failing
       - name: Unit tests
-        run: make all phpvendors test-cli test-copyright test-lib test-nomos test-ojo test-reuser test-scheduler test-spdx2 test-unifiedreport test-www
+        env:
+          PGHOST: 127.0.0.1
+          PGPORT: 5432
+        run: |
+          make all phpvendors test-cli test-copyright test-lib test-monk \
+          test-nomos test-ojo test-reuser test-scheduler test-spdx2 \
+          test-unifiedreport test-www
diff --git a/install/db/licenseRef.json b/install/db/licenseRef.json
diff --git a/src/cli/exportLicenseRefUsingSPDX.php b/src/cli/exportLicenseRefUsingSPDX.php
@@ -1,6 +1,6 @@
 <?php
 /***********************************************************
- Copyright (C) 2019 Siemens AG
+ Copyright (C) 2019,2021 Siemens AG
 
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
@@ -42,6 +42,10 @@ function startProcessingLicenseData()
     $updateExisting = '';
     $addNewLicense = '';
     $newLicenseRefData = array();
+    $scanList = array(
+      'licenses' => 'https://spdx.org/licenses/licenses.json',
+      'exceptions' => 'https://spdx.org/licenses/exceptions.json'
+    );
     $usage = "Usage: " . basename($argv[0]) . " [options]
 
       Create new licenseref.json file.  Options are:
@@ -64,10 +68,6 @@ function startProcessingLicenseData()
           For type 'licenses' URL is : $scanList[licenses]
 
           For type 'exceptions' URL is : $scanList[exceptions]";
-    $scanList = array(
-    'licenses' => 'https://spdx.org/licenses/licenses.json',
-    'exceptions' => 'https://spdx.org/licenses/exceptions.json'
-    );
 
     $options = getopt("hcEen", array("type:","url:"));
     /* get type and url if exists if not set them to empty */
@@ -203,10 +203,10 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
             !empty($updateExisting)
           )
          ) {
-        $existingLicenseRefData[$licenseIdCheck]['rf_fullname'] = $getCurrentData[$this->mapArrayData[$type][2]];
-        $existingLicenseRefData[$licenseIdCheck]['rf_text'] = $getCurrentData[$this->mapArrayData[$type][1]];
+        $existingLicenseRefData[$licenseIdCheck]['rf_fullname'] = $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][2]]);
+        $existingLicenseRefData[$licenseIdCheck]['rf_text'] = $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][1]]);
         $existingLicenseRefData[$licenseIdCheck]['rf_url'] = $getCurrentData['seeAlso'][0];
-        $existingLicenseRefData[$licenseIdCheck]['rf_notes'] = (array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : $existingLicenseRefData[$licenseIdCheck]['rf_notes']);
+        $existingLicenseRefData[$licenseIdCheck]['rf_notes'] = $this->replaceUnicode((array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : $existingLicenseRefData[$licenseIdCheck]['rf_notes']));
         echo "INFO: license ".$getCurrentData[$this->mapArrayData[$type][0]]." updated\n\n";
       }
       if (!is_numeric($licenseIdCheck) &&
@@ -218,7 +218,7 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
          ) {
         $existingLicenseRefData[] = array(
           'rf_shortname' => $getCurrentData[$this->mapArrayData[$type][0]],
-          'rf_text' =>  $getCurrentData[$this->mapArrayData[$type][1]],
+          'rf_text' =>  $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][1]]),
           'rf_url' =>  $getCurrentData['seeAlso'][0],
           'rf_add_date' => null,
           'rf_copyleft' => null,
@@ -227,7 +227,7 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
           'rf_FSFfree' => null,
           'rf_GPLv2compatible' => null,
           'rf_GPLv3compatible' => null,
-          'rf_notes' => (array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : null),
+          'rf_notes' => $this->replaceUnicode((array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : null)),
           'rf_Fedora' => null,
           'marydone' => "f",
           'rf_active' => "t",
@@ -243,6 +243,35 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
     }
     return $existingLicenseRefData;
   }
+
+  /**
+   * Replace common unicode characters with ASCII for consistent results.
+   *
+   * @param string $text Input text
+   * @return string Input with characters replaced
+   */
+  private function replaceUnicode($text)
+  {
+    $search = [
+      '\u00a0',  // no break space
+      '\u2018',  // Left single quote
+      '\u2019',  // Right single quote
+      '\u201c',  // Left double quote
+      '\u201d',  // Right double quote
+      '\u2013',  // em dash
+    ];
+
+    $replace = [
+      " ",
+      "'",
+      "'",
+      '"',
+      '"',
+      "-",
+    ];
+
+    return str_replace($search, $replace, $text);
+  }
 }
 $obj = new exportLicenseRef();
 echo $obj->startProcessingLicenseData();
diff --git a/src/monk/agent_tests/Functional/bulkTest.php b/src/monk/agent_tests/Functional/bulkTest.php
@@ -138,15 +138,6 @@ private function getHeartCount($output)
     }
   }
 
-  /** @group Functional */
-  public function testDatabaseSetup()
-  {
-    $this->setUpTables();
-    $this->setUpRepo();
-
-    $this->rmRepo();
-  }
-
   /** @group Functional */
   public function testRunTwoIndependentMonkBulkScans()
   {

diff --git a/src/scheduler/agent_tests/Functional/Functional.py b/src/scheduler/agent_tests/Functional/Functional.py
@@ -32,7 +32,7 @@
 from xml.dom.minidom import parseString
 from xml.dom import Node
 from optparse import OptionParser
-import ConfigParser
+import configparser
 import subprocess
 import functools
 import signal
@@ -121,7 +121,7 @@ def __init__(self, node):
     self.defines['pids'] = {}
 
     # get variable definitions
-    for i in xrange(definitions.length):
+    for i in range(definitions.length):
       if definitions.item(i).name not in self.defines:
         self.defines[definitions.item(i).name] = self.substitute(definitions.item(i).value, defNode)
 
@@ -275,7 +275,7 @@ def createAction(self, node):
     Returns the new action
     """
     def action_wrapper(action, node, doc, dest):
-      print '.',# node.nodeName,
+      print('.', end=' ')# node.nodeName,
       return action(node, doc, dest)
 
     if not hasattr(self, node.nodeName):
@@ -398,8 +398,8 @@ def loadConf(self, node, doc, dest):
     """
     dir = self.required(node, 'directory')
 
-    config = ConfigParser.ConfigParser()
-    config.readfp(open(dir + "/fossology.conf"))
+    config = configparser.ConfigParser()
+    config.read_file(open(dir + "/fossology.conf"))
 
     self.defines["FOSSOLOGY"] = {}
     self.defines["BUILD"] = {}
@@ -408,7 +408,7 @@ def loadConf(self, node, doc, dest):
     self.defines["FOSSOLOGY"]["path"]  = config.get("FOSSOLOGY", "path")
     self.defines["FOSSOLOGY"]["depth"] = config.get("FOSSOLOGY", "depth")
 
-    config.readfp(open(dir + "/VERSION"))
+    config.read_file(open(dir + "/VERSION"))
 
     self.defines["BUILD"]["VERSION"] = config.get("BUILD", "VERSION")
     self.defines["BUILD"]["COMMIT_HASH"] = config.get("BUILD", "COMMIT_HASH")
@@ -448,7 +448,7 @@ def loop(self, node, doc, dest):
           tests  += ret[0]
           failed += ret[1]
     else:
-      for i in xrange(int(iterations)):
+      for i in range(int(iterations)):
         self.defines[varname] = str(i)
         for action in actions:
           ret = action(doc, dest)
@@ -598,12 +598,12 @@ def performTests(self, suiteNode, document, fname):
     tests        = 0
     totalasserts = 0
 
-    print "start up",
+    print("start up", end=' ')
     for action in self.setup:
       while action(None, None)[1] != 0:
         time.sleep(5)
 
-    print "tests",
+    print("tests", end=' ')
     for test in self.tests:
       assertions = 0
       testNode = document.createElement("testcase")
@@ -627,7 +627,7 @@ def performTests(self, suiteNode, document, fname):
 
       suiteNode.appendChild(testNode)
 
-    print " clean up",
+    print(" clean up", end=' ')
     for action in self.cleanup:
       action(None, None)
 
@@ -637,7 +637,7 @@ def performTests(self, suiteNode, document, fname):
     suiteNode.setAttribute("failures", str(failures))
     suiteNode.setAttribute("tests", str(tests))
     suiteNode.setAttribute("assertions", str(totalasserts))
-    print
+    print()
 
 ################################################################################
 ### MAIN #######################################################################
@@ -684,10 +684,10 @@ def main():
     if options.specific and suite.getAttribute("name") != options.specific:
       continue
     if suite.hasAttribute("disable"):
-      print suite.getAttribute("name"),'::','disabled'
+      print(suite.getAttribute("name"),'::','disabled')
       continue
     if options.skipLongTests and suite.hasAttribute("longest") and int(suite.getAttribute("longest"))>int(options.skipLongTests):
-      print suite.getAttribute("name"),'::','expected to run',suite.getAttribute("longest"),'time units'
+      print(suite.getAttribute("name"),'::','expected to run',suite.getAttribute("longest"),'time units')
       continue
     suiteNode = resultsDoc.createElement("testsuite")
     errors = 0
@@ -706,7 +706,7 @@ def main():
       curr.cleanup = cleanup + curr.cleanup
 
       starttime = time.time()
-      print "{0: >15} ::".format(suite.getAttribute("name")),
+      print("{0: >15} ::".format(suite.getAttribute("name")), end=' ')
       if not timeout(functools.partial(curr.performTests, suiteNode, resultsDoc, testFile.name), maxRuntime):
         errors += 1
         errorNode = resultsDoc.createElement("error")

diff --git a/src/spdx2/agent_tests/Functional/fo_report.sql b/src/spdx2/agent_tests/Functional/fo_report.sql
@@ -77,8 +77,8 @@ INSERT INTO pfile VALUES (5, '39C379E9C7F5BB524754C4DEF5FEF135', '840B5882792482
 INSERT INTO pfile VALUES (6, '2702A657B801333C3150BDC8BE642F9B', '798826BF3EB294E5D514ECFA3222CCF09BBCD985', '8df6eb5d69ffe2bf61937d49f3ef72e98213fd09f9ad41c626e419503178bacd', 14554, NULL);
 
 
-INSERT INTO users VALUES (1, 'Default User', 1, 'Default User when nobody is logged in', 'Seed', 'Pass', 0, NULL, 'y', 'public', NULL, NULL, 'simple', NULL, NULL, NULL, 1);
-INSERT INTO users VALUES (2, 'fossy', 1, 'Default Administrator', '14272952581103610285', 'cdd40d0517419e8495a6e40b14369b6a39031581', 10, 'y', 'y', 'public', NULL, NULL, 'simple', NULL, NULL, NULL, 1);
+INSERT INTO users (user_pk, user_name, root_folder_fk, user_desc, user_seed, user_pass, user_perm, user_email, email_notify, upload_visibility, user_agent_list, default_bucketpool_fk, ui_preference, new_upload_group_fk, new_upload_perm, default_folder_fk) VALUES (1, 'Default User', 1, 'Default User when nobody is logged in', 'Seed', 'Pass', 0, NULL, 'y', 'public', NULL, NULL, 'simple', NULL, NULL, 1);
+INSERT INTO users (user_pk, user_name, root_folder_fk, user_desc, user_seed, user_pass, user_perm, user_email, email_notify, upload_visibility, user_agent_list, default_bucketpool_fk, ui_preference, new_upload_group_fk, new_upload_perm, default_folder_fk) VALUES (2, 'fossy', 1, 'Default Administrator', '16294329171791449506', 'b27fd9578d6893916952c2fb74a64bbc9e1bf0b9', 10, 'y', 'y', 'public', 'agent_copyright,agent_mimetype,agent_monk,agent_nomos,agent_pkgagent', 1, '', NULL, NULL, 1);
 
 
 INSERT INTO clearing_decision (clearing_decision_pk, uploadtree_fk, pfile_fk, user_fk, group_fk, decision_type, scope, date_added) VALUES (2, 4, 2, 2, 2, 5, 1, '2015-05-04 11:43:18.276425+02');

diff --git a/src/unifiedreport/agent_tests/Functional/fo_report.sql b/src/unifiedreport/agent_tests/Functional/fo_report.sql
@@ -116,8 +116,8 @@ INSERT INTO pfile VALUES (6, '2702A657B801333C3150BDC8BE642F9B', '798826BF3EB294
 -- Data for Name: users; Type: TABLE DATA; Schema: public; Owner: fossy
 --
 
-INSERT INTO users VALUES (1, 'Default User', 1, 'Default User when nobody is logged in', 'Seed', 'Pass', 0, NULL, 'y', 'public', NULL, NULL, 'simple', NULL, NULL, NULL, 1);
-INSERT INTO users VALUES (2, 'fossy', 1, 'Default Administrator', '14272952581103610285', 'cdd40d0517419e8495a6e40b14369b6a39031581', 10, 'y', 'y', 'public', NULL, NULL, 'simple', NULL, NULL, NULL, 1);
+INSERT INTO users (user_pk, user_name, root_folder_fk, user_desc, user_seed, user_pass, user_perm, user_email, email_notify, upload_visibility, user_agent_list, default_bucketpool_fk, ui_preference, new_upload_group_fk, new_upload_perm, default_folder_fk) VALUES (1, 'Default User', 1, 'Default User when nobody is logged in', 'Seed', 'Pass', 0, NULL, 'y', 'public', NULL, NULL, 'simple', NULL, NULL, 1);
+INSERT INTO users (user_pk, user_name, root_folder_fk, user_desc, user_seed, user_pass, user_perm, user_email, email_notify, upload_visibility, user_agent_list, default_bucketpool_fk, ui_preference, new_upload_group_fk, new_upload_perm, default_folder_fk) VALUES (2, 'fossy', 1, 'Default Administrator', '16294329171791449506', 'b27fd9578d6893916952c2fb74a64bbc9e1bf0b9', 10, 'y', 'y', 'public', 'agent_copyright,agent_mimetype,agent_monk,agent_nomos,agent_pkgagent', 1, '', NULL, NULL, 1);
 
 
 --