import { expect, test } from "@jest/globals";
import { Document } from "../../document.js";
import { MozillaReadabilityTransformer } from "../mozilla_readability.js";
test("Test HTML to text transformer", async () => {
const webpageText = `
🦜️🔗 LangChain
🦜️🔗 LangChain
⚡ Building applications with LLMs through composability ⚡
As an open source project in a rapidly developing field, we are extremely open to contributions.
`;
const documents = [
new Document({
pageContent: webpageText,
}),
new Document({
pageContent: "Mitochondria is the powerhouse of the cell.
",
metadata: { reliable: false },
}),
];
const transformer = new MozillaReadabilityTransformer();
const newDocuments = await transformer.transformDocuments(documents);
expect(newDocuments.length).toBe(2);
expect(newDocuments[0].pageContent.length).toBeLessThan(webpageText.length);
expect(newDocuments[1].pageContent).toBe(
"Mitochondria is the powerhouse of the cell."
);
expect(newDocuments[1].metadata).toEqual({ reliable: false });
});