OpenCorporates

OpenCorporates is a leading open data project, which aims to make available information on every company in the world. I helped them to obtain some of it.

Information on companies can be enormously valuable for a range of purposes, from businesses assessing the creditworthiness of their customers, to tracing the ownership of disputed assets, and unravelling complex money-laundering schemes. But it is normally scattered and inconsistently structured.

I wrote and maintained bots to scrape data about financial services licenses each month from more than 20 regulators' websites in 18 countries, and to transform that data into a standard format for inclusion in the database. I attended monthly hack nights and contributed to the project Slack. I was the most prolific community contributor.

Below are details of some of the more complicated bots I wrote, with links to the source and output on Github:

Original source data

The general approach for each bot was to find a list of the categories of license covered, and an index of all the companies licensed under each category. The bot would then work through each entity in turn.

I overcame a number of issues when working through the various sites, including:

It was often the case that information would be spread across a number of tables. Each bot was set up to parse a specific template, with exception handling. These bots would pull back authoritative data on more than 50,000 companies each month.

Example output

Each bot returned two sets of JSON records, as requested by OpenCorporates. Firstly, it would provide a raw version of all the data it scraped. Secondly, it would provide a transformed version which was in line with the schemas used by the platform and suitable for inclusion in the database.

Below is the raw and transformed output for an example investment firm in the Netherlands:

Raw Transformed
{
	"category": "Financial service provider",
	"Handelsnaam": "ABVO Assuranti\u00ebn en Onroerende Zaken B.V., ABVO Assuranti\u00ebn en Onroerende Zaken B.V.",
	"Vergunningen bemiddelen": [
		{
			"Product": "Betaalrekeningen",
			"Begindatum": "13 mrt 2012",
			"Financiele Dienst": "Bemiddelen (2:80)"
		},
		{
			"Product": "Consumptief krediet",
			"Begindatum": "26 sep 2006",
			"Financiele Dienst": "Bemiddelen (2:80)"
		},
		{
			"Product": "Hypothecair krediet",
			"Begindatum": "26 sep 2006",
			"Financiele Dienst": "Bemiddelen (2:80)"
		},
		{
			"Product": "Inkomensverzekeringen",
			"Begindatum": "01 jan 2014",
			"Financiele Dienst": "Bemiddelen (2:80)"
		}, 
		{
			"Product": "Schadeverzekeringen particulier",
			"Begindatum": "01 jan 2014",
			"Financiele Dienst": "Bemiddelen (2:80)"
		}, 
		{
			"Product": "Schadeverzekeringen zakelijk", 
			"Begindatum": "01 jan 2014",
			"Financiele Dienst": "Bemiddelen (2:80)"
		},
		{
			"Product": "Spaarrekeningen", 
			"Begindatum": "23 apr 2010", 
			"Financiele Dienst": "Bemiddelen (2:80)"
		}, 
		{
			"Product": "Vermogen", 
			"Begindatum": "01 jan 2014", 
			"Financiele Dienst": "Bemiddelen (2:80)"
		}, 
		{
			"Product": "Zorgverzekeringen", 
			"Begindatum": "01 jan 2014", 
			"Financiele Dienst": "Bemiddelen (2:80)"
		}
	],
	"name": "ABVO Assuranti\u00ebn en Onroerende Zaken B.V.", 
	"country": "Nederland", 
	"KvK": "12030408", 
	"Vergunningen adviseren": [
		{
			"Product": "Betaalrekeningen",
			"Begindatum": "13 mrt 2012", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Consumptief krediet", 
			"Begindatum": "26 sep 2006", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Hypothecair krediet", 
			"Begindatum": "26 sep 2006", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Inkomensverzekeringen", 
			"Begindatum": "01 jan 2014", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Schadeverzekeringen particulier", 
			"Begindatum": "01 jan 2014", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Schadeverzekeringen zakelijk", 
			"Begindatum": "01 jan 2014", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Spaarrekeningen", 
			"Begindatum": "23 apr 2010", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Vermogen", 
			"Begindatum": "01 jan 2014", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Zorgverzekeringen",
			"Begindatum": "01 jan 2014", 
			"Financiele Dienst": "Adviseren (2:75)"
		}, 
		{
			"Product": "Deelnemingsrechten in beleggingsinstellingen", 
			"Begindatum": "01 jul 2008", 
			"Financiele Dienst": "Adviseren (2:96)"
		}
	], 
	"Vergunningnummer": "12012957", 
	"source": "Netherlands Authority for the Financial Markets (AFM)", 
	"Contact": [
		{
			"Vergunningnummer:": "ABVO Assuranti\u00ebn en Onroerende Zaken B.V."
		}, 
		{
			"Vergunningnummer:": "ABVO Assuranti\u00ebn en Onroerende Zaken B.V., ABVO Assuranti\u00ebn en Onroerende Zaken B.V."
		}, 
		{
			"Vergunningnummer:": "REUVER"
		}, 
		{
			"Vergunningnummer:": "Nederland"
		}, 
		{
			"Vergunningnummer:": "12030408"
		}
	], 
	"Beleidsbepalers": [
		{
			"Naam": "AMH van Deur", 
			"Vergunningnummer": "12012957"
		}, 
		{
			"Naam": "GGJ Hesen", 
			"Vergunningnummer": "12012957"
		}
	], 
	"licence_number": "12012957", 
	"sample_date": "2015-10-30", 
	"source_url": "http://www.afm.nl/nl/professionals/registers/alle-huidige-registers/financieel-dienstverlener.aspx?id=B28539C0-E3FA-DF11-BB5A-005056BE6692", 
	"Statutaire naam": "ABVO Assuranti\u00ebn en Onroerende Zaken B.V.", 
	"Plaats": "REUVER"
}
{
	"category": ["Financial"],
	"confidence": "HIGH", 
	"jurisdiction_of_licence": "Netherlands", 
	"licence_holder": {
		"entity_properties": {
			"mailing_address": "REUVER",
			"jurisdiction": "Nederland", 
			"name": "ABVO Assuranti\u00ebn en Onroerende Zaken B.V.", 
			"alternative_names": [
				{
					"type": "trading", 
					"company_name": "ABVO Assuranti\u00ebn en Onroerende Zaken B.V., ABVO Assuranti\u00ebn en Onroerende Zaken B.V."
				}
			]
		}, 
		"entity_type": "unknown"
	}, 
	"licence_number": "12012957", 
	"sample_date": "2015-10-30", 
	"licence_issuer": {
		"jurisdiction": "Netherlands", 
		"name": "Netherlands Authority for the Financial Markets (AFM)"
	}, 
	"permissions": [
		{
			"activity_name": "Financial service provider", 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Betaalrekeningen", 
			"other_attributes": {
				"start_date": "2012-03-13"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Consumptief krediet", 
			"other_attributes": {
				"start_date": "2006-09-26"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Hypothecair krediet", 
			"other_attributes": {
				"start_date": "2006-09-26"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Inkomensverzekeringen",
			"other_attributes":	{
				"start_date": "2014-01-01"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Schadeverzekeringen particulier", 
			"other_attributes": {
				"start_date": "2014-01-01"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Schadeverzekeringen zakelijk", 
			"other_attributes": {
				"start_date": "2014-01-01"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Spaarrekeningen", 
			"other_attributes": {
				"start_date": "2010-04-23"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Vermogen", 
			"other_attributes": {
				"start_date": "2014-01-01"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Zorgverzekeringen", 
			"other_attributes": {
				"start_date": "2014-01-01"
			}, 
			"permission_type": "operating"
		}, 
		{
			"activity_name": "Advice permit - Deelnemingsrechten in beleggingsinstellingen", 
			"other_attributes": {
				"start_date": "2008-07-01"
			}, 
			"permission_type": "operating"
		}
	], 
	"other_attributes": {
		"policymakers": [
		{
			"position": "Policymaker", 
			"name": "AMH van Deur"
		}, 
		{
			"position": "Policymaker", 
			"name": "GGJ Hesen"
		}
		]
	},
	"source_url": "http://www.afm.nl/nl/professionals/registers/alle-huidige-registers/financieel-dienstverlener.aspx?id=B28539C0-E3FA-DF11-BB5A-005056BE6692"
}