diff --git a/package.json b/package.json index d129a6dd..f655d330 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@types/geojson": "^7946.0.12", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", + "cheerio": "1.0.0-rc.12", "copyfiles": "^2.4.1", "docx": "^8.2.4", "eslint": "^8.51.0", @@ -67,10 +68,10 @@ "lodash.omit": "^4.5.0", "lodash.set": "^4.3.2", "lodash.startcase": "^4.4.0", + "marked": "^9.1.5", "prettier": "^3.0.3", "react": "^18.2.0", "react-dom": "^18.2.0", - "striptags": "^3.2.0", "type-fest": "^4.4.0", "uuid": "^9.0.1", "zod": "^3.22.4" @@ -89,12 +90,12 @@ "@types/lodash.set": "^4.3.7", "@types/lodash.startcase": "^4.4.8", "@types/node": "^18.16.19", + "@types/react": "^18.2.17", "@types/react-dom": "^18.2.14", "@types/uuid": "^9.0.6", - "esbuild": "^0.19.5", - "@types/react": "^18.2.17", "@typescript-eslint/eslint-plugin": "^6.2.0", "@typescript-eslint/parser": "^6.8.0", + "esbuild": "^0.19.5", "esbuild-jest": "^0.5.0", "eslint": "^8.46.0", "eslint-plugin-simple-import-sort": "^10.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e774d19..4d433d78 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ dependencies: ajv-formats: specifier: ^2.1.1 version: 2.1.1(ajv@8.12.0) + cheerio: + specifier: 1.0.0-rc.12 + version: 1.0.0-rc.12 copyfiles: specifier: ^2.4.1 version: 2.4.1 @@ -74,6 +77,9 @@ dependencies: lodash.startcase: specifier: ^4.4.0 version: 4.4.0 + marked: + specifier: ^9.1.5 + version: 9.1.5 prettier: specifier: ^3.0.3 version: 3.0.3 @@ -83,9 +89,6 @@ dependencies: react-dom: specifier: ^18.2.0 version: 18.2.0(react@18.2.0) - striptags: - specifier: ^3.2.0 - version: 3.2.0 type-fest: specifier: ^4.4.0 version: 4.4.0 @@ -2261,6 +2264,10 @@ packages: pascalcase: 0.1.1 dev: true + /boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + dev: false + /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2395,6 +2402,30 @@ packages: engines: {node: '>=10'} dev: true + /cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + dev: false + + /cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + dev: false + /ci-info@2.0.0: resolution: {integrity: sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==} dev: true @@ -2606,6 +2637,21 @@ packages: shebang-command: 2.0.0 which: 2.0.2 + /css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.1.1 + dev: false + + /css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + dev: false + /csstype@3.1.2: resolution: {integrity: sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==} @@ -2728,6 +2774,33 @@ packages: csstype: 3.1.2 dev: false + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + dev: false + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + dev: false + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dev: false + /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} dev: true @@ -2754,6 +2827,11 @@ packages: once: 1.4.0 dev: true + /entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + dev: false + /error-ex@1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: @@ -3389,6 +3467,15 @@ packages: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} dev: true + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + dev: false + /human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -4457,6 +4544,12 @@ packages: object-visit: 1.0.1 dev: true + /marked@9.1.5: + resolution: {integrity: sha512-14QG3shv8Kg/xc0Yh6TNkMj90wXH9mmldi5941I2OevfJ/FQAFLEwtwU2/FfgSAOMlWHrEukWSGQf8MiVYNG2A==} + engines: {node: '>= 16'} + hasBin: true + dev: false + /memoizee@0.4.15: resolution: {integrity: sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==} dependencies: @@ -4667,6 +4760,12 @@ packages: path-key: 4.0.0 dev: true + /nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + dependencies: + boolbase: 1.0.0 + dev: false + /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -4780,6 +4879,19 @@ packages: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + /parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + dev: false + + /parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + dependencies: + entities: 4.5.0 + dev: false + /pascalcase@0.1.1: resolution: {integrity: sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==} engines: {node: '>=0.10.0'} @@ -5385,10 +5497,6 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} - /striptags@3.2.0: - resolution: {integrity: sha512-g45ZOGzHDMe2bdYMdIvdAfCQkCTDMGBazSw1ypMowwGIee7ZQ5dU0rBJ8Jqgl+jAKIv4dbeE1jscZq9wid1Tkw==} - dev: false - /strnum@1.0.5: resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==} dev: false diff --git a/src/export/bops/index.ts b/src/export/bops/index.ts index e2b3fae5..37bb11f4 100644 --- a/src/export/bops/index.ts +++ b/src/export/bops/index.ts @@ -1,6 +1,7 @@ +import { load } from "cheerio"; import isEmpty from "lodash.isempty"; import isNil from "lodash.isnil"; -import striptags from "striptags"; +import { marked } from "marked"; import { Passport } from "../../models/passport"; import { getResultData } from "../../models/result"; @@ -264,10 +265,7 @@ export function formatProposalDetails({ } if (crumb.autoAnswered) metadata.auto_answered = true; if (node.data?.policyRef) { - metadata.policy_refs = [ - // remove html tags - { text: striptags(node.data?.policyRef as string) }, - ]; + metadata.policy_refs = parsePolicyRefs(node.data.policyRefs as string); } return metadata; })(); @@ -286,6 +284,17 @@ export function formatProposalDetails({ }; } +export const parsePolicyRefs = (markdownOrHTML: string) => { + const htmlString = marked.parse(markdownOrHTML); + const $ = load(htmlString); + const policyRefs = $("a").map((_index, el) => ({ + text: $(el).prop("textContent")?.trim() ?? undefined, + url: $(el).attr("href"), + })); + + return policyRefs.toArray(); +}; + export function computeBOPSParams({ sessionId, flow, diff --git a/src/export/bops/mocks/payload.ts b/src/export/bops/mocks/payload.ts index 5fbb6840..a2970243 100644 --- a/src/export/bops/mocks/payload.ts +++ b/src/export/bops/mocks/payload.ts @@ -1134,7 +1134,12 @@ export const mockExpectedBOPSPayload = { metadata: { policy_refs: [ { - text: "Town and Country Planning Act 1990, Part 7, Section 191 & Section 192", + text: "Town and Country Planning Act 1990, Part 7, Section 191", + url: "https://www.legislation.gov.uk/ukpga/1990/8/section/191", + }, + { + text: "Section 192", + url: "https://www.legislation.gov.uk/ukpga/1990/8/section/192", }, ], }, @@ -1149,7 +1154,12 @@ export const mockExpectedBOPSPayload = { metadata: { policy_refs: [ { - text: "Town and Country Planning Act 1990 (Section 55), The Town and Country Planning (General Permitted Development) (England) Order 2015", + text: "Town and Country Planning Act 1990 (Section 55)", + url: "https://www.legislation.gov.uk/ukpga/1990/8/section/55", + }, + { + text: "The Town and Country Planning (General Permitted Development) (England) Order 2015", + url: "https://www.legislation.gov.uk/uksi/2015/596/contents", }, ], }, @@ -1165,6 +1175,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Town and Country Planning Act 1990, Section 55 (2)", + url: "https://www.legislation.gov.uk/ukpga/1990/8/section/55", }, ], auto_answered: true, @@ -1312,6 +1323,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Greater London Authority Act 1999", + url: "https://www.legislation.gov.uk/ukpga/1999/29/section/346", }, ], auto_answered: true, @@ -1328,6 +1340,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Greater London Authority Act 1999", + url: "https://www.legislation.gov.uk/ukpga/1999/29/section/346", }, ], }, @@ -1352,6 +1365,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Greater London Authority Act 1999", + url: "https://www.legislation.gov.uk/ukpga/1999/29/section/346", }, ], }, @@ -1368,6 +1382,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Greater London Authority Act 1999", + url: "https://www.legislation.gov.uk/ukpga/1999/29/section/346", }, ], auto_answered: true, @@ -1384,6 +1399,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Greater London Authority Act 1999", + url: "https://www.legislation.gov.uk/ukpga/1999/29/section/346", }, ], }, @@ -1399,6 +1415,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Greater London Authority Act 1999", + url: "https://www.legislation.gov.uk/ukpga/1999/29/section/346", }, ], }, @@ -1414,6 +1431,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "Greater London Authority Act 1999", + url: "https://www.legislation.gov.uk/ukpga/1999/29/section/346", }, ], }, @@ -1486,6 +1504,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "The Town and Country Planning (Development Management Procedure) (England) Order 2015", + url: "https://www.legislation.gov.uk/uksi/2015/595/article/39/made", }, ], }, @@ -1500,7 +1519,8 @@ export const mockExpectedBOPSPayload = { metadata: { policy_refs: [ { - text: "[Town and Country Planning Act 1990 Section 171B](https://www.legislation.gov.uk/ukpga/1990/8/section/171B)", + text: "Town and Country Planning Act 1990 Section 171B", + url: "https://www.legislation.gov.uk/ukpga/1990/8/section/171B", }, ], auto_answered: true, @@ -1621,6 +1641,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012 Schedule 1, Part 2", + url: "https://www.legislation.gov.uk/uksi/2012/2920/contents", }, ], }, @@ -1668,7 +1689,20 @@ export const mockExpectedBOPSPayload = { metadata: { policy_refs: [ { - text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012, Regulation 14UK Statutory Instruments 2012 No. 2920 Regulation 4, Equalities Act 2010, Section 6 Children Act 1989, Part 3", + text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012, Regulation 14", + url: "https://www.legislation.gov.uk/uksi/2012/2920/regulation/14", + }, + { + text: "UK Statutory Instruments 2012 No. 2920 Regulation 4", + url: "https://www.legislation.gov.uk/uksi/2012/2920/regulation/4/made", + }, + { + text: "Equalities Act 2010, Section 6", + url: "https://www.legislation.gov.uk/ukpga/2010/15/section/6", + }, + { + text: "Children Act 1989, Part 3", + url: "https://www.legislation.gov.uk/ukpga/1989/41/part/III", }, ], }, @@ -1685,6 +1719,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012, Regulation 9", + url: "https://www.legislation.gov.uk/uksi/2012/2920/regulation/9", }, ], }, @@ -1722,6 +1757,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012 Chapter 2, Paragraph 3", + url: "https://www.legislation.gov.uk/uksi/2012/2920/schedule/1", }, ], auto_answered: true, @@ -1738,6 +1774,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012 - Regulation 11", + url: "https://www.legislation.gov.uk/uksi/2012/2920/regulation/11", }, ], auto_answered: true, @@ -1755,6 +1792,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012 Chapter 2, Paragraph 10", + url: "https://www.legislation.gov.uk/uksi/2012/2920/schedule/1", }, ], }, @@ -1783,6 +1821,7 @@ export const mockExpectedBOPSPayload = { policy_refs: [ { text: "The Town and Country Planning (Fees for Applications, Deemed Applications, Requests and Site Visits) (England) Regulations 2012 - Regulation 11", + url: "https://www.legislation.gov.uk/uksi/2012/2920/regulation/11", }, ], auto_answered: true, diff --git a/src/types/export.ts b/src/types/export.ts index 7a0facde..210bac36 100644 --- a/src/types/export.ts +++ b/src/types/export.ts @@ -19,6 +19,7 @@ export interface PlanXExportData { export interface PolicyRefs { text: string; + url?: string; } export interface ResponseObject {