Skip to content
Snippets Groups Projects
Commit da8e6be9 authored by echicken's avatar echicken :chicken:
Browse files

Some work toward a better ingestion process.

parent 028f549c
No related branches found
No related tags found
No related merge requests found
import fs from 'fs';
import path from 'path';
import puppeteer from 'puppeteer';
import { ingestText } from './ai.js';
const URL = 'https://synchro.net/docs/jsobjs.html';
const OUT_BASE = path.join(path.resolve(), 'data', 'jsobjs');
const OUT_FILE = path.join(OUT_BASE, 'jsobjs.json');
interface Method {
name: string,
returns: string,
usage: string,
version: string,
description: string,
};
interface Property {
name: string,
type: string,
version: string,
description: string,
};
interface Scope {
methods: Record<string, Method>,
properties: Record<string, Property>,
};
type JSObjs = Record<string, Scope>;
export async function extractData(): Promise<void> {
const browser = await puppeteer.launch({
executablePath: '/usr/bin/google-chrome',
args: [
'--no-sandbox',
],
});
const page = await browser.newPage();
await page.goto(URL);
await page.setViewport({ width: 1080, height: 1024 });
const sections = await page.evaluate(() => {
const sections: JSObjs = {};
const elements = document.querySelectorAll('body > font > table');
if (elements === undefined || elements === null) return;
for (const element of elements) {
const scope = element.querySelector('caption > b > tt').innerHTML;
if (sections[scope] === undefined) sections[scope] = { methods: {}, properties: {} };
const type = element.querySelector('caption > b > a').innerHTML;
const rows = element.querySelectorAll('tbody > tr');
if (type.trim() === 'methods') {
sections[scope]['methods'] = {};
for (const row of rows) {
const td = row.querySelectorAll('td');
if (td.length < 5) continue;
sections[scope]['methods'][td[0].innerText] = {
name: td[0].innerText,
returns: td[1].innerText,
usage: td[2].innerText,
version: td[3].innerText,
description: td[4].innerText,
};
}
} else {
sections[scope]['properties'] = {};
for (const row of rows) {
const td = row.querySelectorAll('td');
if (td.length < 4) continue;
sections[scope]['properties'][td[0].innerText] = {
name: td[0].innerText,
type: td[1].innerText,
version: td[2].innerText,
description: td[3].innerText,
};
}
}
}
return sections;
});
fs.writeFileSync(OUT_FILE, JSON.stringify(sections));
await browser.close();
}
export async function ingest(): Promise<void> {
await extractData();
const jsobjs: JSObjs = JSON.parse(fs.readFileSync(OUT_FILE, 'utf8'));
for (const scope in jsobjs) {
const prefix = `${scope}${scope === 'global' ? ' ' : '.'}`;
let str = `The following document describes the "${scope}" object of the Synchronet javascript object model. This object is available to scripts running in the javascript runtime environment of Synchronet BBS.\r\n`;
str += `\r\n---\r\n\r\nThe following is a list of methods on the "${scope}" object:\r\n`;
for (const method in jsobjs[scope].methods) {
const _prefix = `${prefix}${method}`;
str += '\r\n';
str += `The return value type of the "${_prefix}" method is: "${jsobjs[scope].methods[method].returns}"\r\n`;
str += `The usage example for the "${_prefix}" method is: "${jsobjs[scope].methods[method].usage}"\r\n`;
str += `The description of the "${_prefix}" method is: "${jsobjs[scope].methods[method].description}"\r\n`;
str += `The "${_prefix}" method was introduced in Synchronet "${jsobjs[scope].methods[method].version}"\r\n`;
}
str += `\r\n---\r\n\r\nThe following is a list of properties of the "${scope}" object:\r\n`;
for (const property in jsobjs[scope].properties) {
const _prefix = `${prefix}${property}`;
str += '\r\n';
str += `The value type of the "${_prefix}" property is: "${jsobjs[scope].properties[property].type}"\r\n`;
str += `The description of the "${_prefix}" property is: "${jsobjs[scope].properties[property].description}"\r\n`;
str += `The "${_prefix}" property was introduced in Synchronet "${jsobjs[scope].properties[property].version}"\r\n`;
}
fs.writeFileSync(path.join(OUT_BASE, `${scope}.txt`), str);
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment