update for 2024.10 version

code4fukui · Oct 18, 2024 · ad790a7 · ad790a7
1 parent ee8b9d5
commit ad790a7
Show file tree

Hide file tree

Showing 7 changed files with 809 additions and 1,572 deletions.
diff --git a/saj-memberlist.csv b/saj-memberlist.csv
diff --git a/saj-memberlist0.csv b/saj-memberlist0.csv
diff --git a/src/fetchList.js b/src/fetchList.js
diff --git a/src/makeCSV.js b/src/makeCSV.js
diff --git a/src/parseList.js b/src/parseList.js
diff --git a/src/scrape.js b/src/scrape.js
@@ -1,2 +1 @@
 await import("./scrapeList.js");
-await import("./makeCSV.js");
diff --git a/src/scrapeList.js b/src/scrapeList.js
@@ -1,14 +1,37 @@
 import { fetchOrLoad, HTMLParser, CSV, nextTag, prevTag, table2json, table2csv, sleep } from "https://code4fukui.github.io/scrapeutil/scrapeutil.js";
-import { fetchList } from "./fetchList.js";
 
-const fn = "../saj-memberlist0.csv";
-const url = "https://archive.saj.or.jp/memberlist/all.html";
+const fn = "../saj-memberlist.csv";
 
-const html = await fetchOrLoad(url);
-const dom = HTMLParser.parse(html);
+const list = [];
+for (let i = 0;; i += 20) {
+  const url = "https://www.saj.or.jp/M10/M1001AL/corporate_name/asc/" + i;
+  //const url = "https://www.saj.or.jp/M10/M1001AL";
 
-const link = dom.querySelector(".content_main iframe").getAttribute("src");
-console.log(link);
+  const html = await fetchOrLoad(url);
+  const dom = HTMLParser.parse(html);
 
-const list = await fetchList(link);
+  const dls = dom.querySelectorAll(".table_list li dl");
+  if (dls.length <= 1) break;
+  for (let j = 1; j < dls.length; j++) {
+    const dl = dls[j];
+    const dtdd = dl.querySelectorAll("dt,dd");
+    const item = {};
+    for (let i = 0; i < dtdd.length; i += 2) {
+      const name = dtdd[i].text.trim();
+      const val = dtdd[i + 1].text.trim();
+      //console.log(name, val);
+      item[name] = val;
+    }
+    list.push({
+      name: item.法人名,
+      pref: item.都道府県,
+      url: item.法人URL,
+      type: item.会員区分,
+    });
+    console.log(list.length)
+  }
+  await sleep(200);
+}
+
+console.log(list, list.length)
 await Deno.writeTextFile(fn, CSV.stringify(list));
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1 @@
		await import("./scrapeList.js");
		await import("./makeCSV.js");