Some work toward a better ingestion process.

src/lib/jsobjs.ts

+import fs from 'fs';
+import path from 'path';
+import puppeteer from 'puppeteer';
+import { ingestText } from './ai.js';
+
+const URL = 'https://synchro.net/docs/jsobjs.html';
+const OUT_BASE = path.join(path.resolve(), 'data', 'jsobjs');
+const OUT_FILE = path.join(OUT_BASE, 'jsobjs.json');
+
+interface Method {
+	name: string,
+	returns: string,
+	usage: string,
+	version: string,
+	description: string,
+};
+
+interface Property {
+	name: string,
+	type: string,
+	version: string,
+	description: string,
+};
+
+interface Scope {
+	methods: Record<string, Method>,
+	properties: Record<string, Property>,
+};
+
+type JSObjs = Record<string, Scope>;
+
+export async function extractData(): Promise<void> {
+
+	const browser = await puppeteer.launch({
+		executablePath: '/usr/bin/google-chrome',
+		args: [
+			'--no-sandbox',
+		],
+	});
+
+	const page = await browser.newPage();
+	await page.goto(URL);
+	await page.setViewport({ width: 1080, height: 1024 });
+	const sections = await page.evaluate(() => {
+		const sections: JSObjs = {};
+		const elements = document.querySelectorAll('body > font > table');
+		if (elements === undefined || elements === null) return;
+		for (const element of elements) {
+			const scope = element.querySelector('caption > b > tt').innerHTML;
+			if (sections[scope] === undefined) sections[scope] = { methods: {}, properties: {} };
+			const type = element.querySelector('caption > b > a').innerHTML;
+			const rows = element.querySelectorAll('tbody > tr');
+			if (type.trim() === 'methods') {
+				sections[scope]['methods'] = {};
+				for (const row of rows) {
+					const td = row.querySelectorAll('td');
+					if (td.length < 5) continue;
+					sections[scope]['methods'][td[0].innerText] = {
+						name: td[0].innerText,
+						returns: td[1].innerText,
+						usage: td[2].innerText,
+						version: td[3].innerText,
+						description: td[4].innerText,
+					};
+				}
+			} else {
+				sections[scope]['properties'] = {};
+				for (const row of rows) {
+					const td = row.querySelectorAll('td');
+					if (td.length < 4) continue;
+					sections[scope]['properties'][td[0].innerText] = {
+						name: td[0].innerText,
+						type: td[1].innerText,
+						version: td[2].innerText,
+						description: td[3].innerText,
+					};
+				}
+			}
+		}
+		return sections;
+	});
+	fs.writeFileSync(OUT_FILE, JSON.stringify(sections));
+	await browser.close();
+}
+
+export async function ingest(): Promise<void> {
+	await extractData();
+	const jsobjs: JSObjs = JSON.parse(fs.readFileSync(OUT_FILE, 'utf8'));
+	for (const scope in jsobjs) {
+		const prefix = `${scope}${scope === 'global' ? ' ' : '.'}`;
+		let str = `The following document describes the "${scope}" object of the Synchronet javascript object model. This object is available to scripts running in the javascript runtime environment of Synchronet BBS.\r\n`;
+		str += `\r\n---\r\n\r\nThe following is a list of methods on the "${scope}" object:\r\n`;
+		for (const method in jsobjs[scope].methods) {
+			const _prefix = `${prefix}${method}`;
+			str += '\r\n';
+			str += `The return value type of the "${_prefix}" method is: "${jsobjs[scope].methods[method].returns}"\r\n`;
+			str += `The usage example for the "${_prefix}" method is: "${jsobjs[scope].methods[method].usage}"\r\n`;
+			str += `The description of the "${_prefix}" method is: "${jsobjs[scope].methods[method].description}"\r\n`;
+			str += `The "${_prefix}" method was introduced in Synchronet "${jsobjs[scope].methods[method].version}"\r\n`;
+		}
+		str += `\r\n---\r\n\r\nThe following is a list of properties of the "${scope}" object:\r\n`;
+		for (const property in jsobjs[scope].properties) {
+			const _prefix = `${prefix}${property}`;
+			str += '\r\n';
+			str += `The value type of the "${_prefix}" property is: "${jsobjs[scope].properties[property].type}"\r\n`;
+			str += `The description of the "${_prefix}" property is: "${jsobjs[scope].properties[property].description}"\r\n`;
+			str += `The "${_prefix}" property was introduced in Synchronet "${jsobjs[scope].properties[property].version}"\r\n`;
+		}
+		fs.writeFileSync(path.join(OUT_BASE, `${scope}.txt`), str);
+	}
+}