colttaine/masterscraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added basic date functionality

Browse source
This commit is contained in:
colttaine 2023-03-06 11:18:13 +11:00
parent d8a4ac2b99
commit 05654beea7
2 changed files with 11 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
masterscraper/__init__.py
View file

@ -116,6 +116,8 @@ class scrape:
key_name.append('country.name')
elif(key.lower().find('year') >=0):
key_name.append('year')
elif(key.lower().find('date') >=0):
key_name.append('date')
else:
tmp_key = key
@ -299,6 +301,12 @@ class scrape:
y2 = self.data[-1][self.data_info[1].index('year')]
if y1 <= y2: key_year.append( '{0}-{1}'.format(y1,y2) )
if y1 > y2: key_year.append( '{0}-{1}'.format(y2,y1) )
elif 'date' in self.data_info[1]:
y1 = self.data[1][self.data_info[1].index('date')].split('-')[0]
y2 = self.data[-1][self.data_info[1].index('date')].split('-')[0]
if y1 <= y2: key_year.append( '{0}-{1}'.format(y1,y2) )
if y1 > y2: key_year.append( '{0}-{1}'.format(y2,y1) )
else:
key_year.append( date.today().strftime('%Y') )
self.data_info.append( key_year )
@ -308,6 +316,7 @@ class scrape:
for key in self.data_info[1]:
if key == 'country.name': self.meta['type'] = 'global'
elif key == 'year': self.meta['type'] = 'historical'
elif key == 'date': self.meta['type'] = 'historical'
elif key == 'us.county.fips': self.meta['type'] = 'regional'
elif key == 'uk.constituency.name': self.meta['type'] = 'regional'
if self.meta['type'] == None: self.meta['type'] = 'unkown'
@ -421,7 +430,7 @@ class scrape:
self.meta['category'] = 'economic'
self.meta['subcategory'] = 'welfare'
elif( search.find('manufatur') >=0 or
elif( search.find('manufactur') >=0 or
search.find('business') >=0 or
search.find('tourism') >=0 ):
self.meta['category'] = 'economic'

2
scrape_single.py
View file

@ -5,7 +5,7 @@ import masterscraper as ms
scrape = ms.scrape('https://www.macrotrends.net/countries/AUT/austria/population-density')
scrape = ms.scrape('https://www.macrotrends.net/countries/ISL/iceland/economic-growth-rate')
scrape.get_meta()
scrape.clean()
scrape.save()