Generating cleanup

mod.ts

@@ -1,2 +1 @@
-import * as Providers from "./src/providers/mod.ts";
-export { Providers };
+export * from "./src/mod.ts";

scripts/fetch-databases.ts

@@ -1,4 +1,4 @@
-import { render } from "https://deno.land/x/eta@v1.6.0/mod.ts";
+import { render } from "https://deno.land/x/eta@v1.14.2/mod.ts";
 import { emptyDir } from "https://deno.land/x/[email protected]/mod.ts";
 
 import { DataResult, ResponseData, WithSodaVersion } from "./util/interfaces.ts";
@@ -29,7 +29,9 @@ const getData = async (url: string) => {
   response.results.forEach((result) => {
     const description = (result.resource.description || "")
       .trim()
-      .split("\n");
+      .split("\n")
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0);
 
     const data: DataResult = {
       name: fixTitle(result.resource.name),
@@ -64,8 +66,14 @@ const renderRemplate = (template: string, item: WithSodaVersion) =>
     autoEscape: false,
     autoTrim: false,
   }) as Promise<string>).then((result) => {
-    const fixed = result.replace(/^(\/\/\r\n){2,}/g, "//\r\n");
-    return fixed;
+    const fixed = result
+      .replace(/^(^\/\/\s?\n)/g, "//\n")
+      .replace(/^(\/\/\s?\r\n){2,}/g, "//\r\n");
+
+    // replace multiple lines that only have a * with a single line
+    const fixed2 = fixed.replace(/(^\s+\*\s?[r\|\n]){1,3}/gm, "$1");
+
+    return fixed2;
   });
 
 const renderAndWrite = async (template: string, item: WithSodaVersion) => {
@@ -122,7 +130,10 @@ const run = async ({ dryRun }: { dryRun?: boolean } = {}) => {
     await Deno.mkdir(outputFolder, { recursive: true });
     // cleanup outputfolder
     await emptyDir(outputFolder);
-    const modRendered = await (render(modTemplate, { data: resultData }, {
+    const modRendered = await (render(modTemplate, {
+      items: resultData,
+      sodaVersion: lastSodaVersion,
+    }, {
       async: true,
       cache: true,
       rmWhitespace: false,

scripts/templates/mod.ts.ejs

@@ -1,2 +1,17 @@
-<% it.data.forEach(function (item) { %>
+export {
+  DataType,
+  Field,
+  Order,
+  Select,
+  SelectAll,
+  SelectGreatest,
+  SelectLeast,
+  SelectRegrIntercept,
+  SelectRegrR2,
+  SelectRegrSlope,
+  SystemFields,
+  Where,
+} from "https://deno.land/x/soda@<%= it.sodaVersion %>/mod.ts";
+
+<% it.items.forEach(function (item) { %>
 export * as <%= item.name %> from "./<%= item.name %>.ts";<% }); %>

scripts/templates/provider.ts.ejs

@@ -17,33 +17,18 @@
 
 import type { AuthOpts, FieldObject, Options } from "https://deno.land/x/soda@<%= it.item.sodaVersion %>/mod.ts";
 import { createQueryWithDataset, DataType, Field } from "https://deno.land/x/soda@<%= it.item.sodaVersion %>/mod.ts";
-export {
-  Order,
-  Select,
-  SelectAll,
-  SelectGreatest,
-  SelectLeast,
-  SelectRegrIntercept,
-  SelectRegrR2,
-  SelectRegrSlope,
-  SystemFields,
-  Where,
-} from "https://deno.land/x/soda@<%= it.item.sodaVersion %>/mod.ts";
-export { DataType, Field };
 
 /**
  * Return Data for <%= it.item.full_name %>
  */
-export interface ResponseData {
-  <% it.item.columns.forEach(function(column) { %>
+export interface ResponseData {<% it.item.columns.forEach(function(column) { %>
   /**
    * ### <%= column.name %>
    *
    * <% column.description.forEach(function(line) { %>
    * <%= line %><% }); %>
    *
    * **Type**: <%= column.datatype %>
-   *
    */
   <%= column.field_name %>?: <%= column.node_type %>;<% }); %>
 }
@@ -58,28 +43,24 @@ export interface ResponseData {
  *
  * > You can use these fieldnames in your queries to filter, group, or sort your data.
  */
-export interface IFields {
-  <% it.item.columns.forEach(function(column) { %>
+export interface IFields {<% it.item.columns.forEach(function(column) { %>
   /**
    * ### <%= column.name %>
    *
    * <% column.description.forEach(function(line) { %>
    * <%= line %><% }); %>
    *
    * **Type**: <%= column.datatype %>
-   *
    */
    <%= column.big_name %>: FieldObject<<%= column.datatypeTemplate %>>;<% }); %>
 };
 
-export const Fields: IFields = {
-  <% it.item.columns.forEach(function(column) { %>
+export const Fields: IFields = {<% it.item.columns.forEach(function(column) { %>
   <%= column.big_name %>: Field("<%= column.field_name %>", <%= column.datatypeTemplate %>),<% }); %>
 };
 
 export const Info = {
-  fields: [
-    <% it.item.columns.forEach(function(column) { %>
+  fields: [<% it.item.columns.forEach(function(column) { %>
     "<%= column.big_name %>",<% }); %>
   ],
   dataset: "<%= it.item.id %>",
@@ -90,7 +71,6 @@ export const Info = {
   api_docs: "https://dev.socrata.com/foundry/<%= it.item.domain %>/<%= it.item.id %>",
 }
 
-
 /**
  * ### <%= it.item.full_name %><% it.item.description.forEach(function(line) { %>
  * <%= line %><% }); %>

scripts/util/mod.ts

@@ -34,7 +34,7 @@ export const pascalCase = (str: string) => {
 
 export const cleanString = (str?: string) => {
   const cleanRegex = /[^\x20-\x7E\u200B-\u200D\uFEFF\uBBBF]/g;
-  const cleanAndTrim = (s: string) => s.replace(cleanRegex, "").trim();
+  const cleanAndTrim = (s: string) => s.replace(cleanRegex, "").replace("\r", "").replace("\n", "").trim();
   const splitted = (str || "").split("\n");
   if (splitted.length === 0) {
     return "";
@@ -76,7 +76,9 @@ export const mapColumns = (res: Result): Array<{
 
   const columns = columns_name.map((n, i) => {
     const name = cleanString(n);
-    const description = cleanString(columns_description[i]).split("\n");
+    const description = cleanString(columns_description[i]).split("\n").map((s) => s.trim()).filter(
+      (s) => s.length > 0
+    );
     const datatype = cleanString(columns_datatype[i]);
     const datatypeTemplate = dataTypeToDataTypeTemplate(datatype);
     const field_name = cleanString(columns_field_name[i]);