Github action for custom metric tests with WPT API

.github/workflows/wpt-test.yml

@@ -0,0 +1,47 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: WebPageTest Test Cases
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: |
+          npm install webpagetest
+          npm install jest
+
+      - name: Run WebPageTest
+        run: |
+          EXPECTED_TESTS=$(for file in $(git diff --name-only --diff-filter=ACMRT \
+            ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | \
+            grep -E "^dist/.*\.js$"); do basename "$file"; done | cut -d\. -f1 | sort | uniq)
+
+          for TEST in ${EXPECTED_TESTS[@]}; do
+            echo "::group::Test case for $TEST"
+            if [ -f "tests/$TEST.test.js" ]; then
+              npm test $TEST
+            else
+              echo "Test case file tests/$TEST.test.js not found"
+              exit 2
+            fi
+            echo "::endgroup::"
+          done
+        env:
+          WPT_API_KEY: ${{ secrets.WPT_API_KEY }}
+
+      - name: Add comment with results
+        uses: mshick/add-pr-comment@v2
+        if: always()
+        with:
+          refresh-message-position: true
+          message-path: test-results.txt

README.md

@@ -25,12 +25,14 @@ return JSON.stringify({
 });
 ```
 
-3. Test your changes on WPT using the workflow below.
+2. Test your changes on WPT using the workflow below.
 
-4. Submit a pull request. Include one or more links to test results in your PR description to verify that the script is working.
+3. Submit a pull request. Include one or more links to test results in your PR description to verify that the script is working.
 
 ## Testing
 
+### Manual testing using webpagetest.org website
+
 To test a custom metric, for example [`doctype.js`](https://github.com/HTTPArchive/legacy.httparchive.org/blob/master/custom_metrics/doctype.js), you can enter the script directly on [webpagetest.org](https://webpagetest.org?debug=1) under the "Custom" tab.
 
 ![image](https://user-images.githubusercontent.com/1120896/59539351-e3ecdd80-8eca-11e9-8b43-76bbd7a12029.png)
@@ -48,6 +50,12 @@ To see the custom metric results, select a run, first click on "Details", and th
 
 For complex metrics like [almanac.js](./dist/almanac.js) you can more easily explore the results by copy/pasting the JSON into your browser console.
 
+### Automated testing using test cases
+
+0. Tests are running using [WPT API wrapper](https://github.com/webpagetest/webpagetest-api) and [Jest Testing Framework](https://jestjs.io/).
+
+1. Create a test file in the [`tests`](./tests) directory with a name corresponding to a custom metrics file, e.g. [`privacy.test.js`](./tests/privacy.test.js) testing [privacy.js](./dist/privacy.js) custom metrics. In test file you need to define websites for WPT test runs and test cases for the custom metric parameters.
+
 ## Linting
 
 On opening a Pull Request we will do some basic linting of JavaScript using [ESLint](https://eslint.org/) through the [GitHub Super-Linter](https://github.com/github/super-linter).

dist/ads.js

@@ -3,7 +3,7 @@
 const ACCOUNT_TYPES = ['direct', 'reseller'];
 const SELLER_TYPES = ['publisher', 'intermediary', 'both'];
 
-const isPresent = (response, endings) => response.ok && endings.find(ending => response.url.endsWith(ending));
+const isPresent = (response, endings) => response.ok && endings.some(ending => response.url.endsWith(ending));
 
 const fetchAndParse = async (url, parser) => {
   const controller = new AbortController();
@@ -22,20 +22,20 @@ const fetchAndParse = async (url, parser) => {
   }
 };
 
-// Extracts status, record count, and record counts respective to relationship.
-// Standard Specification: https://iabtechlab.com/wp-content/uploads/2022/04/Ads.txt-1.1.pdf
+// https://iabtechlab.com/wp-content/uploads/2022/04/Ads.txt-1.1.pdf
 const parseAdsTxt = async (response) => {
-  const content = await response.text();
+  let content = await response.text();
 
   let result = {
     present: isPresent(response, ['/ads.txt', '/app-ads.txt']),
     redirected: response.redirected,
     status: response.status,
   };
 
-  if (result.present) {
+  if (result.present && content) {
     result = {
-      ...result, ...{
+      ...result,
+      ...{
         account_count: 0,
         account_types: {
           direct: {
@@ -54,10 +54,10 @@ const parseAdsTxt = async (response) => {
     };
 
     // Clen up file content
-    file_content = file_content.replace(/#.*$/gm, '');
-    file_content = file_content.replace(/\r/g, '');
+    content = content.replace(/#.*$/gm, '');
+    content = content.replace(/\r/g, '');
 
-    let lines = file_content.split('\n');
+    let lines = content.split('\n');
     result.line_count = lines.length;
 
     for (let line of lines) {
@@ -81,7 +81,7 @@ const parseAdsTxt = async (response) => {
       }
     };
 
-    // Convert Sets to Arrays
+    // Count unique and remove domain Sets for now
     for (let accountType of Object.values(result.account_types)) {
       accountType.domain_count = accountType.domains.size;
       delete accountType.domains // Keeping a list of domains may be valuable for further research, e.g. accountType.domains = [...accountType.domains];
@@ -94,8 +94,7 @@ const parseAdsTxt = async (response) => {
 }
 
 
-//Extracts seller record mertrics.
-//Standard Specification: https://iabtechlab.com/wp-content/uploads/2019/07/Sellers.json_Final.pdf
+// https://iabtechlab.com/wp-content/uploads/2019/07/Sellers.json_Final.pdf
 const parseSellersJSON = async (response) => {
   let content;
   try {
@@ -104,14 +103,15 @@ const parseSellersJSON = async (response) => {
     content = null;
   }
   let result = {
-    present: isPresent(response, ['/ads.txt', '/app-ads.txt']),
+    present: isPresent(response, ['/sellers.json']),
     redirected: response.redirected,
     status: response.status,
   };
 
-  if (result.present) {
+  if (result.present && content) {
     result = {
-      ...result, ...{
+      ...result,
+      ...{
         seller_count: 0,
         seller_types: {
           publisher: {
@@ -132,39 +132,37 @@ const parseSellersJSON = async (response) => {
     };
 
     // Clean up file content
-    file_content_json = JSON.parse(file_content);
-    file_content_json.seller_count = file_content_json.sellers.length;
+    result.seller_count = content.sellers.length;
 
-    for (let seller of file_content_json.sellers) {
+    for (let seller of content.sellers) {
       // Seller records
       let type = seller.seller_type.trim().toLowerCase(),
-        domain = seller.domain.trim();
+        domain = seller.domain.trim().toLowerCase();
       if (Object.keys(result.seller_types).includes(type)) {
         result.seller_types[type].domains.add(domain);
         result.seller_types[type].seller_count += 1;
       }
-      result.seller_count += 1;
 
       // Passthrough
       if (seller.is_passthrough) {
         result.passthrough_count += 1;
       }
     }
-  };
 
-  // Count unique and remove domain Sets for now
-  for (let seller_type of Object.values(result.seller_types)) {
-    seller_type.domain_count = seller_type.domains.size;
-    delete seller_type.domains //seller_type.domains = [...seller_type.domains];
-  }
+    // Count unique and remove domain Sets for now
+    for (let seller_type of Object.values(result.seller_types)) {
+      seller_type.domain_count = seller_type.domains.size;
+      delete seller_type.domains //seller_type.domains = [...seller_type.domains];
+    }
+  };
 
   return result;
 }
 
 return Promise.all([
-  fetchAndParse("/ads.txt", parseAdsTxt),
-  fetchAndParse("/app-ads.txt", parseAdsTxt),
-  fetchAndParse("/sellers.json", parseSellersJSON),
+  fetchAndParse("/ads.txt", parseAdsTxt).catch(e => e),
+  fetchAndParse("/app-ads.txt", parseAdsTxt).catch(e => e),
+  fetchAndParse("/sellers.json", parseSellersJSON).catch(e => e),
 ]).then((all_data) => {
   return JSON.stringify({
     ads: all_data[0],

dist/privacy.js

@@ -39,16 +39,14 @@ return JSON.stringify({
    * Privacy policies
    * Wording sourced from: https://github.com/RUB-SysSec/we-value-your-privacy/blob/master/privacy_wording.json
    * words = privacy_wording.map(country => country.words).filter((v, i, a) => a.indexOf(v) === i).flat().sort().join('|');
-   *
-   * Test site: https://www.theverge.com/
    */
   privacy_wording_links: (() => {
     let words =
       'adatkezelési|adatvédelem|adatvédelmi|andmekaitsetingimused|aviso legal|beskyttelse af personlige oplysninger|cgu|cgv|confidentialitate|confidentialite|confidentialité|confidentialité|confidentialité|confidentialité|confidentialité|confidențialitate|cookie policy|cookie-uri|cookie-urilor|cookiepolitik|cookies|data policy|data policy|data policy|data policy|datapolicy|datapolitik|datenrichtlinie|datenrichtlinie|datenrichtlinie|datenrichtlinie|datenschutz|datenschutz|datenschutz|datenschutz|datenschutzbestimmungen|datenschutzrichtlinie|donnees personelles|gdpr|gegevensbeleid|gegevensbeleid|gizlilik|gizlilik|integritetspolicy|isikuandmete|isikuandmete töötlemise|kasutustingimused|kişisel verilerin korunması|kolačići|konfidencialiteti|konfidentsiaalsuse|kvkk|küpsised|mbrojtja e të dhënave|mentions légales|mentions légales|normativa sui dati|ochrana dat|ochrana osobních údajů|ochrana osobných údajov|ochrana soukromí|ochrana súkromia|ochrana udaju|ochrana údajov|ochrany osobných údajov|osobné údaje|personlige data|personoplysninger|personuppgifter|personvern|persónuvernd|piškotki|piškotkih|podmínky|policy|politica de utilizare|politika e të dhënave|politikat e privatesise|politikat e privatësisë|politique d’utilisation des données|politique d’utilisation des données|politique d’utilisation des données|politique d’utilisation des données|politique d’utilisation des données|política de dados|política de dados|política de datos|política de datos|pravila o upotrebi podataka|privaatsus|privacidad|privacidad|privacidade|privacidade|privacy|privacy|privacy|privacy|privacy|privacy policy|privacybeleid|privacybeleid|privatezza|privatlivspolitik|privatnost|privatnost|privatnosti|privatssphäre|privatumas|privatumo|privatësia|privātuma|privātums|protecció de dades|protecţia datelor|prywatnosci|prywatności|prywatność|regler om fortrolighed|rekisteriseloste|retningslinjer for data|rgpd|sekretess|slapukai|soukromi|soukromí|személyes adatok védelme|súkromie|sīkdatne|sīkdatņu|tietokäytäntö|tietosuoja|tietosuojakäytäntö|tietosuojaseloste|varstvo podatkov|veri i̇lkesi|veri i̇lkesi|veri politikası|vie privée|webbplatsen|yksityisyyden suoja|yksityisyydensuoja|yksityisyys|zasady dotyczące danych|zasebnost|zaštita podataka|zásady ochrany osobných|zásady používání dat|zásady používání dat|zásady využívania údajov|απόρρητο|απόρρητο|πολιτική απορρήτου|πολιτική δεδομένων|προσωπικά δεδομένα|όροι και γνωστοποιήσεις|конфиденциальность|конфіденційність|поверителност|политика за бисквитки|политика за данни|политика использования данных|политика конфиденциальности|политика о подацима|политика о подацима|политика о подацима|политика обработки персональных данных|приватност|приватност|приватност|условия|условия за ползване|מדיניות נתונים|פרטיות|الخصوصية|سياسة البيانات|数据使用政策|數據使用政策|私隱政策|隐私权政策';
     let pattern = new RegExp('\\b(?:' + words + ')\\b', 'ig');
 
     let privacy_links = Array.from(document.querySelectorAll('a')).map(
-      a => ({keywords: a.innerText.match(pattern), text: a.innerText})
+      a => ({ keywords: a.innerText.match(pattern), text: a.innerText })
     ).filter(a => a.keywords); // filter out non-matching texts (keywords = null)
 
     return privacy_links;
@@ -71,7 +69,6 @@ return JSON.stringify({
       if (consentData.present) {
         // Standard command: 'getVendorConsents'
         // cf. https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/CMP%20JS%20API%20v1.1%20Final.md#what-api-will-need-to-be-provided-by-the-cmp-
-        // Test site: ?
         window.__cmp('getVendorConsents', null, (result, success) => {
           if (success) {
             consentData.data = result;
@@ -98,8 +95,6 @@ return JSON.stringify({
   /**
    * IAB Transparency and Consent Framework v2
    * docs v2: https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/TCFv2
-   *
-   * Test site: https://www.rtl.de/
    */
   iab_tcf_v2: (() => {
     let tcData = {
@@ -137,8 +132,6 @@ return JSON.stringify({
   /**
    * IAB US Privacy User Signal Mechanism “USP API”
    * https://github.com/InteractiveAdvertisingBureau/USPrivacy
-   *
-   * Test site: https://www.nfl.com/
    */
   iab_usp: (() => {
     let uspData = {
@@ -161,8 +154,6 @@ return JSON.stringify({
   /**
    * Ads Transparency Spotlight Data Disclosure schema
    * Only for top frame, can't access child frames (same-origin policy)
-   *
-   * Test site: unknown
    */
   ads_transparency_spotlight: (() => {
     // Check `meta` tag cf. https://github.com/Ads-Transparency-Spotlight/documentation/blob/main/implement.md
@@ -178,10 +169,9 @@ return JSON.stringify({
   })(),
 
   /**
-   * FLoC
+   * FLoC (Federated Learning of Cohorts) - deprecated
    *
    * Test site: https://floc.glitch.me/
-   * Test site: https://www.pokellector.com/
    *
    * @todo Check function/variable accesses through string searches (wrappers cannot be used, as the metrics are only collected at the end of the test)
    */
@@ -190,16 +180,12 @@ return JSON.stringify({
   /**
    * Do Not Track (DNT)
    * https://www.eff.org/issues/do-not-track
-   *
-   * Test site: https://www.theverge.com/
    */
   navigator_doNotTrack: testPropertyStringInResponseBodies('navigator.+doNotTrack'),
 
   /**
    * Global Privacy Control
    * https://globalprivacycontrol.org/
-   *
-   * Test site: https://global-privacy-control.glitch.me/
    */
   navigator_globalPrivacyControl: testPropertyStringInResponseBodies(
     'navigator.+globalPrivacyControl'
@@ -255,7 +241,7 @@ return JSON.stringify({
               a => a.tagName === e.tagName && a.referrerpolicy === e.referrerpolicy
             );
             if (!found) {
-              acc.push({...e, count: 1});
+              acc.push({ ...e, count: 1 });
             } else {
               found.count += 1;
             }