From d5488eb46a33d489e63b3c5824979973685b33cf Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Wed, 26 Jul 2023 17:41:33 -0400 Subject: [PATCH 1/4] tools: don't render date of contract signature. ICANN is updating the contract date field in the `gtlds.json` file for many contracts in the near future. Since including the contract date in the `.dat` file as comments isn't very helpful to anyone, and leaving it in during this renewal period would result in a large quantity of auto-pull requests for no-op changes, we choose to omit the date as a quick fix. This commit updates the `newtlds.go` tool to no longer process/render the `DateOfContractSignature`. Unit tests are updated to reflect this change. --- tools/newgtlds.go | 21 +++---- tools/newgtlds_test.go | 121 +++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 85 deletions(-) diff --git a/tools/newgtlds.go b/tools/newgtlds.go index 47c95be0c..fa15613d0 100644 --- a/tools/newgtlds.go +++ b/tools/newgtlds.go @@ -100,8 +100,6 @@ type pslEntry struct { // RegistryOperator holds the name of the registry operator that operates the // gTLD (may be empty). RegistryOperator string - // DateOfContractSignature holds the date the gTLD contract was signed (may be empty). - DateOfContractSignature string // ContractTerminated indicates whether the contract has been terminated by // ICANN. When rendered by the pslTemplate only entries with // ContractTerminated = false are included. @@ -118,7 +116,6 @@ func (e *pslEntry) normalize() { e.ALabel = strings.TrimSpace(e.ALabel) e.ULabel = strings.TrimSpace(e.ULabel) e.RegistryOperator = strings.TrimSpace(e.RegistryOperator) - e.DateOfContractSignature = strings.TrimSpace(e.DateOfContractSignature) // If there is no explicit uLabel use the gTLD as the uLabel. if e.ULabel == "" { @@ -131,23 +128,19 @@ func (e *pslEntry) normalize() { // // If the registry operator field is empty the comment will be of the form: // -// '// : ' +// '// // // If the registry operator field is not empty the comment will be of the form: // -// '// : ' -// -// In both cases the may be empty. +// '// : ' func (e pslEntry) Comment() string { parts := []string{ "//", e.ALabel, - ":", - e.DateOfContractSignature, } // Avoid two trailing spaces if registry operator is empty if e.RegistryOperator != "" { - parts = append(parts, e.RegistryOperator) + parts = append(parts, []string{":", e.RegistryOperator}...) } return strings.Join(parts, " ") } @@ -365,10 +358,10 @@ func filterGTLDs(entries []*pslEntry) []*pslEntry { } // getPSLEntries fetches a list of pslEntry objects (or returns an error) by: -// 1. getting the raw JSON data from the provided url string. -// 2. unmarshaling the JSON data to create pslEntry objects. -// 3. normalizing the pslEntry objects. -// 4. filtering out any legacy or contract terminated gTLDs +// 1. getting the raw JSON data from the provided url string. +// 2. unmarshaling the JSON data to create pslEntry objects. +// 3. normalizing the pslEntry objects. +// 4. filtering out any legacy or contract terminated gTLDs // // If there are no pslEntry objects after unmarshaling the data in step 2 or // filtering the gTLDs in step 4 it is considered an error condition. diff --git a/tools/newgtlds_test.go b/tools/newgtlds_test.go index 51417c26a..f550b4665 100644 --- a/tools/newgtlds_test.go +++ b/tools/newgtlds_test.go @@ -23,31 +23,27 @@ func TestEntryNormalize(t *testing.T) { { name: "already normalized", inputEntry: pslEntry{ - ALabel: "cpu", - ULabel: "ｃｐｕ", - DateOfContractSignature: "2019-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "cpu", + ULabel: "ｃｐｕ", + RegistryOperator: "@cpu's bargain gTLD emporium", }, expectedEntry: pslEntry{ - ALabel: "cpu", - ULabel: "ｃｐｕ", - DateOfContractSignature: "2019-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "cpu", + ULabel: "ｃｐｕ", + RegistryOperator: "@cpu's bargain gTLD emporium", }, }, { name: "extra whitespace", inputEntry: pslEntry{ - ALabel: " cpu ", - ULabel: " ｃｐｕ ", - DateOfContractSignature: " 2019-06-13 ", + ALabel: " cpu ", + ULabel: " ｃｐｕ ", RegistryOperator: " @cpu's bargain gTLD emporium " + "(now with bonus whitespace) ", }, expectedEntry: pslEntry{ - ALabel: "cpu", - ULabel: "ｃｐｕ", - DateOfContractSignature: "2019-06-13", + ALabel: "cpu", + ULabel: "ｃｐｕ", RegistryOperator: "@cpu's bargain gTLD emporium " + "(now with bonus whitespace)", }, @@ -55,15 +51,13 @@ func TestEntryNormalize(t *testing.T) { { name: "no explicit uLabel", inputEntry: pslEntry{ - ALabel: "cpu", - DateOfContractSignature: "2019-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "cpu", + RegistryOperator: "@cpu's bargain gTLD emporium", }, expectedEntry: pslEntry{ - ALabel: "cpu", - ULabel: "cpu", - DateOfContractSignature: "2019-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "cpu", + ULabel: "cpu", + RegistryOperator: "@cpu's bargain gTLD emporium", }, }, } @@ -89,26 +83,25 @@ func TestEntryComment(t *testing.T) { { name: "Full entry", entry: pslEntry{ - ALabel: "cpu", - DateOfContractSignature: "2019-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "cpu", + RegistryOperator: "@cpu's bargain gTLD emporium", }, - expected: "// cpu : 2019-06-13 @cpu's bargain gTLD emporium", + expected: "// cpu : @cpu's bargain gTLD emporium", }, { - name: "Entry with empty contract signature date and operator", + name: "Entry without operator", entry: pslEntry{ ALabel: "cpu", }, - expected: "// cpu : ", + expected: "// cpu", }, { - name: "Entry with empty contract signature and non-empty operator", + name: "Entry with non-empty operator", entry: pslEntry{ ALabel: "cpu", RegistryOperator: "@cpu's bargain gTLD emporium", }, - expected: "// cpu : @cpu's bargain gTLD emporium", + expected: "// cpu : @cpu's bargain gTLD emporium", }, } @@ -162,29 +155,25 @@ func TestGetPSLEntries(t *testing.T) { }{ GTLDs: []pslEntry{ { - ALabel: "ceepeeyou", - DateOfContractSignature: "2099-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "ceepeeyou", + RegistryOperator: "@cpu's bargain gTLD emporium", }, { // NOTE: we include whitespace in this entry to test that normalization // occurs. - ALabel: " cpu ", - ULabel: " ｃｐｕ ", - DateOfContractSignature: " 2019-06-13 ", + ALabel: " cpu ", + ULabel: " ｃｐｕ ", RegistryOperator: " @cpu's bargain gTLD emporium " + "(now with bonus whitespace) ", }, { // NOTE: we include a legacy gTLD here to test that filtering of legacy // gTLDs occurs. - ALabel: "aero", - DateOfContractSignature: "1999-10-31", - RegistryOperator: "Department of Historical Baggage and Technical Debt", + ALabel: "aero", + RegistryOperator: "Department of Historical Baggage and Technical Debt", }, { - ALabel: "terminated", - DateOfContractSignature: "1987-10-31", + ALabel: "terminated", // NOTE: we include a contract terminated = true entry here to test that // filtering of terminated entries occurs. ContractTerminated: true, @@ -197,15 +186,13 @@ func TestGetPSLEntries(t *testing.T) { expectedEntries := []pslEntry{ { - ALabel: "ceepeeyou", - ULabel: "ceepeeyou", - DateOfContractSignature: "2099-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "ceepeeyou", + ULabel: "ceepeeyou", + RegistryOperator: "@cpu's bargain gTLD emporium", }, { - ALabel: "cpu", - ULabel: "ｃｐｕ", - DateOfContractSignature: "2019-06-13", + ALabel: "cpu", + ULabel: "ｃｐｕ", RegistryOperator: "@cpu's bargain gTLD emporium " + "(now with bonus whitespace)", }, @@ -265,21 +252,18 @@ func TestGetPSLEntriesEmptyFilteredResults(t *testing.T) { GTLDs: []pslEntry{ { // NOTE: GTLD matches a legacyGTLDs map entry to ensure filtering. - ALabel: "aero", - DateOfContractSignature: "1999-10-31", - RegistryOperator: "Department of Historical Baggage and Technical Debt", + ALabel: "aero", + RegistryOperator: "Department of Historical Baggage and Technical Debt", }, { - ALabel: "terminated", - DateOfContractSignature: "1987-10-31", + ALabel: "terminated", // NOTE: Setting ContractTerminated to ensure filtering. ContractTerminated: true, }, { - ALabel: "removed", - DateOfContractSignature: "1999-10-31", - RegistryOperator: "Department of Historical Baggage and Technical Debt", - RemovalDate: "2019-08-06", + ALabel: "removed", + RegistryOperator: "Department of Historical Baggage and Technical Debt", + RemovalDate: "2019-08-06", }, }, } @@ -301,22 +285,20 @@ func TestGetPSLEntriesEmptyFilteredResults(t *testing.T) { func TestRenderData(t *testing.T) { entries := []*pslEntry{ { - ALabel: "ceepeeyou", - ULabel: "ceepeeyou", - DateOfContractSignature: "2099-06-13", - RegistryOperator: "@cpu's bargain gTLD emporium", + ALabel: "ceepeeyou", + ULabel: "ceepeeyou", + RegistryOperator: "@cpu's bargain gTLD emporium", }, { - ALabel: "cpu", - ULabel: "ｃｐｕ", - DateOfContractSignature: "2019-06-13", + ALabel: "cpu", + ULabel: "ｃｐｕ", }, } - expectedList := `// ceepeeyou : 2099-06-13 @cpu's bargain gTLD emporium + expectedList := `// ceepeeyou : @cpu's bargain gTLD emporium ceepeeyou -// cpu : 2019-06-13 +// cpu ｃｐｕ ` @@ -675,7 +657,7 @@ func TestProcess(t *testing.T) { // List of new gTLDs imported from https://www.icann.org/resources/registries/gtlds/v2/gtlds.json on 2021-02-07T13:25:56-05:00 // This list is auto-generated, don't edit it manually. -// aaa : 2015-02-26 American Automobile Association, Inc. +// aaa : American Automobile Association, Inc. aaa @@ -688,7 +670,6 @@ aaa "gTLDs": [ { "contractTerminated": false, - "dateOfContractSignature": "2015-02-26", "gTLD": "aaa", "registryOperator": "American Automobile Association, Inc.", "removalDate": null, @@ -703,7 +684,6 @@ aaa "gTLDs": [ { "contractTerminated": false, - "dateOfContractSignature": "2015-02-26", "gTLD": "aaa", "registryOperator": "American Automobile Association, Inc.", "removalDate": null, @@ -711,7 +691,6 @@ aaa }, { "contractTerminated": false, - "dateOfContractSignature": "2014-03-20", "gTLD": "accountants", "registryOperator": "Binky Moon, LLC", "removalDate": null, @@ -731,10 +710,10 @@ aaa // List of new gTLDs imported from https://www.icann.org/resources/registries/gtlds/v2/gtlds.json on 2021-02-10T00:24:14Z // This list is auto-generated, don't edit it manually. -// aaa : 2015-02-26 American Automobile Association, Inc. +// aaa : American Automobile Association, Inc. aaa -// accountants : 2014-03-20 Binky Moon, LLC +// accountants : Binky Moon, LLC accountants From de2ac2dc8ec052aa2a971371defebfb645888dc5 Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Wed, 26 Jul 2023 17:44:51 -0400 Subject: [PATCH 2/4] ci: run Go unit tests as part of `test.yml`. Previously the Go unit tests for the `newgtlds` tool were run as part of the Travis workflow for each branch push. This is helpful to catch breaking changes to the Go tooling _before_ they are merged and used in the `tld-update.yml` workflow. This commit adds the required steps to `test.yml` so that these unit tests are run by GitHub actions similar to the way they were for Travis. --- .github/workflows/test.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff9b70a72..48f18be06 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,14 +6,22 @@ permissions: jobs: make-test: - name: make test + name: Unit tests runs-on: ubuntu-22.04 steps: - name: Check out code uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '^1.15' + + - name: Run Go unit tests + run: go test tools/*.go + - name: Install dependencies run: sudo apt install -y autopoint - - name: Run test + - name: Run makefile tests run: make test From e1b18f62b9918cef201fdd04cdf4151047dcb032 Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Wed, 26 Jul 2023 17:50:59 -0400 Subject: [PATCH 3/4] ci: switch to latest stable go. --- .github/workflows/test.yml | 2 +- .github/workflows/tld-update.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48f18be06..4cff6a98c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '^1.15' + go-version: 'stable' - name: Run Go unit tests run: go test tools/*.go diff --git a/.github/workflows/tld-update.yml b/.github/workflows/tld-update.yml index 2626462f8..93d850965 100644 --- a/.github/workflows/tld-update.yml +++ b/.github/workflows/tld-update.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '^1.15' + go-version: 'stable' - name: Run unit tests run: go test tools/*.go From 3ec6ce31a56b8698d5ef922ceea3d94ed8829f90 Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Wed, 26 Jul 2023 17:51:19 -0400 Subject: [PATCH 4/4] ci: remove stale comment in tld-update.yml The comment about the automation being on-hold was leftover from a period ~two years ago. It isn't accurate, the automation _has_ been running, and it properly handles the header timestamp issue that caused it to be temporarily disabled. --- .github/workflows/tld-update.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tld-update.yml b/.github/workflows/tld-update.yml index 93d850965..7b0cb4945 100644 --- a/.github/workflows/tld-update.yml +++ b/.github/workflows/tld-update.yml @@ -1,4 +1,3 @@ -# Put this automation on hold, until it is able to ignore changes that only update the header timestamp. name: tld-update on: workflow_dispatch: