Natural Language Processing, Adobe PDF Extract, and Deep PDF Intelligence

A look at integrating PDF text extraction with a natural language processor to return deep information about the contents of a PDF.

Nov. 17, 21 · AI Zone · Tutorial

Like (1)

let fields = 'entities,sentiment,facts,records,categories';
let token = 'your private token here';
let url = `https://nl.diffbot.com/v1/?fields=${fields}&token=${token}`;

// the text variable is what you want to parse
let body = [{
	content:text, 
	lang:'en',
	format:'plain text'
}];

let req = await fetch(url, { 
	method:'POST',
	body:JSON.stringify(body),
	headers: { 'Content-Type':'application/json' }
});

let result = await req.json();

const DIFFBOT_KEY = process.env.DIFFBOT_KEY;

const INPUT = './pdfs/';

(async () => {

	let files = await glob(INPUT + '*.pdf');
	console.log(`Going to process ${files.length} PDFs.\n`);

	for(file of files) {
		console.log(`Checking ${file} to see if it has a txt or json file.`);

		// ToDo: This would fail with foo.pdf.pdf. Don't do that.
		let txtFile = file.replace('.pdf', '.txt');
		let jsonFile = file.replace('.pdf', '.json');

		let textExists = await exists(txtFile);
		let text;
		if(!textExists) {
			console.log(`We do not have ${txtFile} and need to make it.`);
			text = await getText(file);
			await fs.writeFile(txtFile, text);
			console.log('I have saved the extracted PDF text.');
		} else {
			console.log(`The text file ${txtFile} already exists.`);
			text = await fs.readFile(txtFile,'utf8');
		}

		let jsonExists = await exists(jsonFile);
		if(!jsonExists) {
			console.log(`We do not have ${jsonFile} and need to make it.`);
			let data = await getData(text);
			await fs.writeFile(jsonFile, JSON.stringify(data));
			console.log('I have saved the parsed data from the text.');
		} else console.log(`The data file ${jsonFile} already exists.`);

	}

})();

async function getText(pdf) {
	const credentials = PDFServicesSdk.Credentials
			.serviceAccountCredentialsBuilder()
			.fromFile('pdftools-api-credentials.json')
			.build();

	// Create an ExecutionContext using credentials
	const executionContext = PDFServicesSdk.ExecutionContext.create(credentials);

	// Build extractPDF options
	const options = new PDFServicesSdk.ExtractPDF.options.ExtractPdfOptions.Builder()
		.addElementsToExtract(
			PDFServicesSdk.ExtractPDF.options.ExtractElementType.TEXT
		)
		.build()

	// Create a new operation instance.
	const extractPDFOperation = PDFServicesSdk.ExtractPDF.Operation.createNew(),
		input = PDFServicesSdk.FileRef.createFromLocalFile(
			pdf,
			PDFServicesSdk.ExtractPDF.SupportedSourceFormat.pdf
		);

	extractPDFOperation.setInput(input);
	extractPDFOperation.setOptions(options);

	let outputZip = './' + nanoid() + '.zip';
	let result = await extractPDFOperation.execute(executionContext);
	await result.saveAsFile(outputZip);
	let zip = new AdmZip(outputZip);
	let data = JSON.parse(zip.readAsText('structuredData.json'));
	let text = data.elements.filter(e => e.Text).reduce((result, e) => {
		return result + e.Text + '\n';
	},'');

	await fs.unlink(outputZip);
	return text;
}

async function getData(text) {
	let fields = 'entities,sentiment,facts,records,categories';
	let token = DIFFBOT_KEY;
	let url = `https://nl.diffbot.com/v1/?fields=${fields}&token=${token}`;
	let body = [{
		content:text, 
		lang:'en',
		format:'plain text'
	}];

	let req = await fetch(url, { 
		method:'POST',
		body:JSON.stringify(body),
		headers: { 'Content-Type':'application/json' }
	});

	return await req.json();
}

{
	"name": "Sarah Clark",
	"confidence": 0.99999547,
	"salience": 0.976258,
	"sentiment": 0,
	"isCustom": false,
	"allUris": [],
	"allTypes": [
		{
			"name": "person",
			"diffbotUri": "https://diffbot.com/entity/E4aFoJie0MN6dcs_yDRFwXQ",
			"dbpediaUri": "http://dbpedia.org/ontology/Person"
		}
	],
	"mentions": [
		{
			"text": "Sarah Clark",
			"beginOffset": 425,
			"endOffset": 436,
			"confidence": 0.99984574
		},
		{
			"text": "Sarah Clark",
			"beginOffset": 38923,
			"endOffset": 38934,
			"confidence": 0.99999547
		}
	]
}

{
	"name": "Microsoft",
	"confidence": 0.9998447,
	"salience": 0,
	"sentiment": 0,
	"isCustom": false,
	"allUris": [],
	"allTypes": [
		{
			"name": "organization",
			"diffbotUri": "https://diffbot.com/entity/EN1ClYEdMMQCxB6AWTkT3mA",
			"dbpediaUri": "http://dbpedia.org/ontology/Organisation"
		}
	],
	"mentions": [
		{
			"text": "Microsoft",
			"beginOffset": 35999,
			"endOffset": 36008,
			"confidence": 0.9998447
		},
		{
			"text": "Microsoft",
			"beginOffset": 38848,
			"endOffset": 38857,
			"confidence": 0.9988986
		}
	]
}

"categories": {
	"iabv1": [
		{
			"id": "IAB19",
			"name": "Technology & Computing",
			"confidence": 0.99088717
		}
	],
	"iabv2": [
		{
			"id": "596",
			"name": "Technology & Computing",
			"confidence": 0.99088717
		}
	]
}

const INPUT = './pdfs/';
const OUTPUT = './pdfdata.json';

(async () => {

	let result = [];

	let files = await glob(INPUT + '*.pdf');

	for(file of files) {
		console.log(`Checking ${file} to see if it has a json file.`);

		let jsonFile = file.replace('.pdf', '.json');

		let jsonExists = await exists(jsonFile);
		if(jsonExists) {
			let json = JSON.parse(await fs.readFile(jsonFile, 'utf8'));
			let people = gatherPeople(json);
			let categories = gatherCategories(json);
			result.push({
				pdf: file, 
				people, 
				categories
			});
		} else console.log(`The data file ${jsonFile} didn't exist so we are skipping.`);
	}

	await fs.writeFile(OUTPUT, JSON.stringify(result));
	console.log(`Done and written to ${OUTPUT}.`);

})();

async function exists(p) {
	try {
		await fs.stat(p);
		return true;
	} catch(e) {
		return false;
	}
}

function gatherPeople(data) {

	let people = data[0].entities.filter(ent => {
		return ent.allTypes.some(type => {
			return type.name === 'person';
		});
	}).map(person => {
		return person.name;
	});

	//https://stackoverflow.com/a/43046408/52160
	return [...new Set(people)];
}

function gatherCategories(data) {

	let categories = [];
	if(data[0].categories['iabv1']) {
		data[0].categories.iabv1.forEach(c => {
			categories.push(c.name);
		});
	}

	if(data[0].categories['iabv2']) {
		data[0].categories.iabv2.forEach(c => {
			categories.push(c.name);
		});
	}

	//https://stackoverflow.com/a/43046408/52160
	return [...new Set(categories)];
}