Added macrotrend scraping
This commit is contained in:
parent
bd916decd5
commit
1246dc373a
7 changed files with 299 additions and 26 deletions
6
conf/macrotrends.txt
Normal file
6
conf/macrotrends.txt
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
https://www.macrotrends.net/countries/CHN/china/population
|
||||
https://www.macrotrends.net/countries/CHN/china/healthcare-spending
|
||||
https://www.macrotrends.net/countries/CHN/china/life-expectancy
|
||||
https://www.macrotrends.net/countries/CHN/china/unemployment-rate
|
||||
https://www.macrotrends.net/countries/CHN/china/gdp-per-capita
|
||||
https://www.macrotrends.net/countries/BRA/brazil/gdp-per-capita
|
||||
|
|
@ -43,12 +43,16 @@ class scrape:
|
|||
|
||||
#--------[ Scrape Constructor Object ]--------#
|
||||
def __init__(self, url):
|
||||
|
||||
print('\n[{0}]'.format(url))
|
||||
|
||||
self.meta = {
|
||||
"name" : None,
|
||||
"description" : None,
|
||||
"units" : None,
|
||||
"year" : None,
|
||||
"notes" : [],
|
||||
"id" : None,
|
||||
"type" : None,
|
||||
"scope" : None,
|
||||
"category" : None,
|
||||
|
|
@ -83,6 +87,9 @@ class scrape:
|
|||
#--------[ Get Metadata ]--------#
|
||||
def get_meta(self):
|
||||
|
||||
# Break if scrape contains no data
|
||||
if len(self.data) <= 1: return(1)
|
||||
|
||||
# Process Name
|
||||
self.meta['name'] = self.meta['name'].lower()
|
||||
self.meta['name'] = re.sub('and\ dependencies ','',self.meta['name'])
|
||||
|
|
@ -103,10 +110,10 @@ class scrape:
|
|||
key.lower().find('dependency') >=0 ):
|
||||
key_name.append('country.name')
|
||||
elif(key.lower().find('year') >=0):
|
||||
key_name.append('country.name')
|
||||
key_name.append('year')
|
||||
else:
|
||||
tmp_key = self.meta['name']
|
||||
tmp_key = tmp_key + ' ' + key
|
||||
|
||||
tmp_key = key
|
||||
tmp_key = tmp_key.lower()
|
||||
|
||||
tmp_key = re.sub('\[.*\]', '', tmp_key)
|
||||
|
|
@ -120,6 +127,10 @@ class scrape:
|
|||
|
||||
tmp_key = tmp_key.strip()
|
||||
tmp_key = tmp_key.replace(' ','.')
|
||||
|
||||
if tmp_key != self.meta['name'].lower().replace(' ','.'):
|
||||
tmp_key = self.meta['name'].lower().replace(' ','.') + '.' + tmp_key
|
||||
|
||||
key_name.append( tmp_key )
|
||||
self.data_info.append( key_name )
|
||||
|
||||
|
|
@ -170,6 +181,11 @@ class scrape:
|
|||
for key in self.data_info[0]:
|
||||
if re.match('\d\d\d\d', key):
|
||||
key_year.append( key )
|
||||
elif 'year' in self.data_info[1]:
|
||||
y1 = self.data[1][self.data_info[1].index('year')]
|
||||
y2 = self.data[-1][self.data_info[1].index('year')]
|
||||
if y1 <= y2: key_year.append( '{0}-{1}'.format(y1,y2) )
|
||||
if y1 > y2: key_year.append( '{0}-{1}'.format(y2,y1) )
|
||||
else:
|
||||
key_year.append( date.today().strftime('%Y') )
|
||||
self.data_info.append( key_year )
|
||||
|
|
@ -201,22 +217,99 @@ class scrape:
|
|||
|
||||
# Get Category
|
||||
search = self.meta['name'].join(self.data_info[0]).lower().strip()
|
||||
|
||||
#--------[ Geographic ]--------#
|
||||
if( search.find('area') >=0 or
|
||||
search.find('km2') >=0):
|
||||
self.meta['category'] = 'geographic'
|
||||
self.meta['subcategory'] = 'area'
|
||||
elif( search.find('depression') >= 0 or
|
||||
|
||||
#--------[ Demographic ]--------
|
||||
elif( search.find('population') >=0 ):
|
||||
self.meta['category'] = 'demogrpahic'
|
||||
|
||||
elif( search.find('birth') >=0 or
|
||||
search.find('fertility') >=0 ):
|
||||
self.meta['category'] = 'demogrpahic'
|
||||
self.meta['subcategory'] = 'fertility'
|
||||
|
||||
#--------[ Health ]--------#
|
||||
elif( search.find('life expectancy') >=0 or
|
||||
search.find('death') >=0 or
|
||||
search.find('suicide') >=0 or
|
||||
search.find('mortality') >=0 ):
|
||||
self.meta['category'] = 'health'
|
||||
self.meta['subcategory'] = 'mortality'
|
||||
|
||||
elif( search.find('depression') >=0 or
|
||||
search.find('anxiety') >=0 ):
|
||||
self.meta['category'] = 'health'
|
||||
self.meta['subcategory'] = 'psychology'
|
||||
elif( search.find('economic') >= 0 or
|
||||
search.find('gdp') >=0 ):
|
||||
|
||||
elif( search.find('smoking') >= 0 or
|
||||
search.find('alcohol') >=0 ):
|
||||
self.meta['category'] = 'health'
|
||||
self.meta['subcategory'] = 'drugs'
|
||||
|
||||
#--------[ Economic ]--------#
|
||||
elif( search.find('gdp') >=0 and
|
||||
search.find('trade') <0 and
|
||||
search.find('health') <0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'gdp'
|
||||
elif( search.find('development') >= 0 or
|
||||
|
||||
elif( search.find('gni') >=0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'gni'
|
||||
|
||||
elif( search.find('debt') >=0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'debt'
|
||||
|
||||
elif( search.find('inflation') >=0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'inflation'
|
||||
|
||||
elif( search.find('health') >=0 and
|
||||
search.find('spend') >=0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'welfare'
|
||||
|
||||
elif( search.find('manufature') >=0 or
|
||||
search.find('business') >=0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'business'
|
||||
|
||||
elif( search.find('import') >=0 or
|
||||
search.find('export') >=0 or
|
||||
search.find('invest') >=0 or
|
||||
search.find('tarrif') >=0 or
|
||||
search.find('trade') >=0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'trade'
|
||||
|
||||
elif( search.find('unemployment') >=0 or
|
||||
search.find('labor') >=0 ):
|
||||
self.meta['category'] = 'economic'
|
||||
self.meta['subcategory'] = 'labor-force'
|
||||
|
||||
#--------[ Education ]--------#
|
||||
elif( search.find('education') >=0 or
|
||||
search.find('literacy') >=0 ):
|
||||
self.meta['category'] = 'education'
|
||||
|
||||
#--------[ Development ]--------#
|
||||
elif( search.find('development') >=0 or
|
||||
search.find('competitive') >=0 ):
|
||||
self.meta['category'] = 'technology'
|
||||
self.meta['subcategory'] = 'development'
|
||||
self.meta['category'] = 'development'
|
||||
|
||||
#--------[ Crime ]--------#
|
||||
elif( search.find('crime') >=0 or
|
||||
search.find('homocide') >=0 or
|
||||
search.find('murder') >=0 ):
|
||||
self.meta['category'] = 'development'
|
||||
|
||||
#--------[ Uncategorised ]--------#
|
||||
else:
|
||||
self.meta['category'] = 'uncategorised'
|
||||
|
||||
|
|
@ -232,6 +325,10 @@ class scrape:
|
|||
|
||||
#--------[ Clean Scrape Data ]--------#
|
||||
def clean(self):
|
||||
|
||||
# Break if scrape contains no data
|
||||
if len(self.data) <= 1: return(1)
|
||||
|
||||
for x in range(1, len(self.data)):
|
||||
for y in range(0, len(self.data[x])):
|
||||
self.data[x][y] = self.data[x][y]
|
||||
|
|
@ -259,7 +356,6 @@ class scrape:
|
|||
if self.data[x][y].is_integer():
|
||||
self.data[x][y] = int(self.data[x][y])
|
||||
|
||||
|
||||
# Convert non-entries to null
|
||||
if isinstance(self.data[x][y], str):
|
||||
if( self.data[x][y].lower().find('not determined') >= 0 or
|
||||
|
|
@ -272,11 +368,12 @@ class scrape:
|
|||
self.data[x][y] = None
|
||||
|
||||
|
||||
|
||||
#--------[ Save Scrape Data ]--------#
|
||||
def save(self):
|
||||
|
||||
print('\n', self.meta['sources'])
|
||||
|
||||
# Break if scrape contains no data
|
||||
if len(self.data) <= 1: return(1)
|
||||
|
||||
key_main = 0
|
||||
for i in range(0, len(self.data_info[1])):
|
||||
|
|
@ -291,17 +388,22 @@ class scrape:
|
|||
|
||||
#--------[ Generate Filename ]--------#
|
||||
filename = self.data_info[1][key_data].replace('.','-')
|
||||
filepath = 'data/{0}/{1}'.format(self.meta['type'], self.meta['category'])
|
||||
if self.meta['subcategory'] != None: filepath = filepath + '/' + self.meta['subcategory']
|
||||
if len(self.data[0]) > 4:
|
||||
filepath = filepath + '/' + self.meta['name'].lower().replace(' ','-')
|
||||
|
||||
if not os.path.exists(filepath):
|
||||
os.makedirs(filepath)
|
||||
filepath = 'data/{0}'.format(self.meta['type'])
|
||||
if self.meta['type'] == 'historical': filepath += '/' + self.meta['scope'].lower().replace(' ','-')
|
||||
filepath += '/{0}'.format(self.meta['category'])
|
||||
if self.meta['subcategory'] != None: filepath += '/' + self.meta['subcategory']
|
||||
if len(self.data[0]) > 4:
|
||||
filepath += '/' + self.meta['name'].lower().replace(' ','-')
|
||||
|
||||
fullpath = filepath + '/' + filename + '.json'
|
||||
|
||||
|
||||
#--------[ Check File Directory ]--------#
|
||||
if not os.path.exists(filepath):
|
||||
os.makedirs(filepath)
|
||||
|
||||
|
||||
#--------[ Open File ]--------#
|
||||
f = open(fullpath, "w")
|
||||
f.write('{\n')
|
||||
|
|
@ -310,6 +412,8 @@ class scrape:
|
|||
#--------[ Update Metadata ]--------#
|
||||
self.meta['units'] = self.data_info[2][key_data]
|
||||
self.meta['year'] = self.data_info[4][key_data]
|
||||
|
||||
if self.meta['scope'] == None:
|
||||
self.meta['scope'] = self.data_info[5][key_data]
|
||||
|
||||
#--------[ Write Metadata ]
|
||||
|
|
@ -343,13 +447,30 @@ class scrape:
|
|||
|
||||
#--------[ Write Actual Data ]--------#
|
||||
f.write(' "data" : [\n')
|
||||
f.write(' ["{0}","{1}"],\n'.format(self.data_info[1][key_main], self.data_info[1][key_data]))
|
||||
|
||||
if self.meta['type'] == 'historical':
|
||||
f.write(' ["{0}","{1}"],\n'.format(
|
||||
self.data_info[1][key_main],
|
||||
self.meta['id'] + '.' + self.data_info[1][key_data])
|
||||
)
|
||||
else:
|
||||
f.write(' ["{0}","{1}"],\n'.format(
|
||||
self.data_info[1][key_main],
|
||||
self.data_info[1][key_data])
|
||||
)
|
||||
|
||||
for row in self.data[1:]:
|
||||
if row[key_data] != None:
|
||||
f.write(' ["{0}",{1}]'.format(row[key_main], row[key_data]))
|
||||
else:
|
||||
f.write(' ["{0}",null]'.format(row[key_main]))
|
||||
col_a = row[key_main]
|
||||
col_b = row[key_data]
|
||||
|
||||
if isinstance(col_a, str): col_a = '"{0}"'.format(col_a)
|
||||
if isinstance(col_b, str): col_b = '"{0}"'.format(col_b)
|
||||
|
||||
if col_a == None: col_a = 'null'
|
||||
if col_b == None: col_b = 'null'
|
||||
|
||||
f.write(' [{0},{1}]'.format(col_a, col_b))
|
||||
|
||||
if row != self.data[-1]: f.write(',\n')
|
||||
else: f.write('\n')
|
||||
f.write(' ]\n')
|
||||
|
|
|
|||
118
masterscraper/macrotrends/__init__.py
Normal file
118
masterscraper/macrotrends/__init__.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import requests
|
||||
import pandas as pd
|
||||
import re
|
||||
from bs4 import BeautifulSoup
|
||||
from datetime import date
|
||||
|
||||
|
||||
|
||||
def getpage(url):
|
||||
#--------[ Get Page From URL ]--------#
|
||||
headers = {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET',
|
||||
'Access-Control-Allow-Headers': 'Content-Type',
|
||||
'Access-Control-Max-Age': '3600',
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'
|
||||
}
|
||||
page = requests.get(url, headers)
|
||||
soup = BeautifulSoup(page.content, 'html.parser')
|
||||
return soup
|
||||
|
||||
|
||||
|
||||
def scrapelist():
|
||||
soup = getpage('https://www.macrotrends.net/countries/topic-overview')
|
||||
|
||||
# Get URL list of global metrics
|
||||
links = []
|
||||
for table in soup.find_all('div', class_='col-xs-3'):
|
||||
for link in table.find_all('a'):
|
||||
links.append('https://www.macrotrends.net' + link['href'])
|
||||
|
||||
# Get full country list for each global metric
|
||||
full_list = []
|
||||
for link in links:
|
||||
soup = getpage(link)
|
||||
table = soup.find('div', class_='col-xs-12')
|
||||
for url in table.find_all('a'):
|
||||
full_list.append('https://www.macrotrends.net/' + url['href'])
|
||||
print(url['href'])
|
||||
#break
|
||||
|
||||
print('\nScraping {0} datasets from MacroTrends\n'.format( len(full_list) ))
|
||||
|
||||
return full_list
|
||||
|
||||
|
||||
|
||||
|
||||
def scrape(url, meta, data):
|
||||
#--------[ Get Page From URL ]--------#
|
||||
soup = getpage(url)
|
||||
|
||||
|
||||
#--------[ Get Metadata ]--------#
|
||||
url_parts = url.split('/')
|
||||
|
||||
meta['name'] = url_parts[-1].replace('-',' ').title()
|
||||
|
||||
meta['description'] = soup.find('h1').text
|
||||
|
||||
meta['authors'].append( soup.find('span', string='Data Source: ').next_sibling.text )
|
||||
|
||||
meta['sources'].append( url )
|
||||
|
||||
meta['scope'] = url_parts[-2].replace('-',' ').title()
|
||||
|
||||
meta['id'] = url_parts[-3].lower()
|
||||
|
||||
|
||||
|
||||
#--------[ Extract Table ]--------#
|
||||
table = soup.find('div', class_='col-xs-6')
|
||||
table = table.find('table', class_='historical_data_table')
|
||||
|
||||
# Get Table Headings
|
||||
for tr in table.find_all('tr'):
|
||||
row = [ th.text.strip() for th in tr.find_all('th')]
|
||||
if len(row) > 1:
|
||||
data.append( row )
|
||||
|
||||
# Get Table Data
|
||||
for tr in table.find_all('tr'):
|
||||
row = [ td.text.strip() for td in tr.find_all('td')]
|
||||
if len(row) > 1:
|
||||
data.append( row )
|
||||
|
||||
#--------[ Process Table ]--------
|
||||
|
||||
# Delete rows with incorrect number of variables
|
||||
key = 0
|
||||
key_len = len(data)
|
||||
while key < key_len:
|
||||
if len(data[key]) != len(data[0]):
|
||||
data.pop(key)
|
||||
key = key-1
|
||||
key = key+1
|
||||
key_len = len(data)
|
||||
|
||||
# Delete unwanted table columns
|
||||
key = 0
|
||||
key_len = len(data[0])
|
||||
while key < key_len:
|
||||
flag = False
|
||||
if data[0][key].lower().find('rank') >=0: flag = True
|
||||
if data[0][key].lower().find('change') >=0: flag = True
|
||||
if data[0][key].lower().find('notes') >=0: flag = True
|
||||
if data[0][key].lower().find('gap') >=0: flag = True
|
||||
if data[0][key].lower().find('Δ') >=0: flag = True
|
||||
if data[0][key].lower().find('growth') >=0: flag = True
|
||||
if flag:
|
||||
for i in range(0, len(data)):
|
||||
data[i].pop(key)
|
||||
key = key-1
|
||||
key = key+1
|
||||
key_len = len(data[0])
|
||||
|
|
@ -7,9 +7,8 @@ from bs4 import BeautifulSoup
|
|||
from datetime import date
|
||||
|
||||
|
||||
def scrape(url, meta, data):
|
||||
|
||||
|
||||
def getpage(url):
|
||||
#--------[ Get Page From URL ]--------#
|
||||
headers = {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
|
|
@ -20,6 +19,12 @@ def scrape(url, meta, data):
|
|||
}
|
||||
page = requests.get(url, headers)
|
||||
soup = BeautifulSoup(page.content, 'html.parser')
|
||||
return soup
|
||||
|
||||
|
||||
def scrape(url, meta, data):
|
||||
#--------[ Get Page From URL ]--------#
|
||||
soup = getpage(url)
|
||||
|
||||
|
||||
#--------[ Get Metadata ]--------#
|
||||
|
|
|
|||
12
scrape_all.py
Normal file
12
scrape_all.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
|
||||
import masterscraper as ms
|
||||
|
||||
|
||||
scrapelist = ms.macrotrends.scrapelist()
|
||||
for url in scrapelist:
|
||||
scrape = ms.scrape(url)
|
||||
scrape.get_meta()
|
||||
scrape.clean()
|
||||
scrape.save()
|
||||
11
scrape_single.py
Normal file
11
scrape_single.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
|
||||
import masterscraper as ms
|
||||
|
||||
|
||||
|
||||
scrape = ms.scrape('https://www.macrotrends.net/countries/TUR/turkey/population')
|
||||
scrape.get_meta()
|
||||
scrape.clean()
|
||||
scrape.save()
|
||||
Loading…
Reference in a new issue