File:Comparison gender life expectancy CIA factbook.svg

import re, os, urllib2, time, datetime, collections

data_oldss = [line.split('|') for line in '''\
-1|WORLD|69|67|71.1|7323187457|-
-20|EUROPEAN UNION|80.2|77.4|83.2|515052778|-
20|China|75.5|73.5|77.9|1373541278|ea
10|India|68.5|67.3|69.8|1266883598|as
25|USA|79.8|77.5|82.1|323995528|na
|Indonesia|72.7|70.1|75.5|258316051|ea
|Brazil|73.8|70.2|77.5|205823665|sa
|Pakistan|67.7|65.8|69.8|201995540|as
-20|Nigeria|53.4|52.4|54.5|186053386|af
|Bangladesh|73.2|71|75.4|156186882|as
-10|Russia|70.8|65|76.8|142355415|eu
1|Japan|85|81.7|88.5|126702133|ea
|Mexico|75.9|73.1|78.8|123166749|na
|Philippines|69.2|65.7|72.9|102624209|ea
|Ethiopia|62.2|59.8|64.7|102374044|af
|Vietnam|73.4|70.9|76.2|95261021|ea
|Egypt|72.7|71.4|74.2|94666993|af
|Iran|71.4|69.8|73.1|82801633|me
-15|DR Congo|57.3|55.8|58.9|81331050|af
|Germany|80.7|78.4|83.1|80722792|eu
|Turkey|74.8|72.5|77.3|80274604|me
|Thailand|74.7|71.5|78|68200824|ea
|France|81.8|78.7|85.1|66836154|eu
12|UK|80.7|78.5|83|64430428|eu
|Italy|82.2|79.6|85|62007540|eu
|Burma|66.6|64.2|69.2|56890418|ea
|South Africa|63.1|61.6|64.6|54300704|af
|Tanzania|62.2|60.8|63.6|52482726|af
|Korea, South|82.4|79.3|85.8|50924172|ea
|Spain|81.7|78.7|84.9|48563476|eu
|Colombia|75.7|72.6|79|47220856|sa
|Kenya|64|62.6|65.5|46790758|af
|Ukraine|71.8|67.1|76.9|44209733|eu
|Argentina|77.1|74|80.4|43886748|sa
|Algeria|76.8|75.5|78.2|40263711|af
|Poland|77.6|73.7|81.7|38523261|eu
|Uganda|55.4|54|56.9|38319241|af
|Iraq|74.9|72.6|77.2|38146025|me
|Sudan|64.1|62|66.3|36729501|af
|Canada|81.9|79.2|84.6|35362905|na
|Morocco|76.9|73.8|80.1|33655786|af
-15|Afghanistan|51.3|49.9|52.7|33332025|as
|Malaysia|75|72.2|78|30949962|ea
|Venezuela|75.8|72.7|78.9|30912302|sa
|Peru|73.7|71.7|75.9|30741062|sa
|Uzbekistan|73.8|70.7|77|29473614|ca
|Nepal|70.7|70.1|71.3|29033914|as
|Saudi Arabia|75.3|73.2|77.4|28160273|me
|Yemen|65.5|63.4|67.8|27392779|me
|Ghana|66.6|64.1|69.1|26908262|af
|Mozambique|53.3|52.6|54.1|25930150|af
|Korea, North|70.4|66.6|74.5|25115311|ea
|Madagascar|65.9|64.4|67.4|24430325|af
|Cameroon|58.5|57.1|59.9|24360803|af
|Cote d'Ivoire|58.7|57.5|59.9|23740424|af
|Taiwan|80.1|77|83.5|23464787|ea
|Australia|82.2|79.8|84.8|22992654|oc
|Sri Lanka|76.8|73.3|80.4|22235000|as
|Romania|75.1|71.7|78.8|21599736|eu
|Angola|56|54.8|57.2|20172332|af
|Burkina Faso|55.5|53.4|57.6|19512533|af
|Niger|55.5|54.3|56.8|18638600|af
|Malawi|61.2|59.2|63.2|18570321|af
|Kazakhstan|70.8|65.6|75.7|18360353|ca
|Chile|78.8|75.7|81.9|17650114|sa
|Mali|55.8|53.9|57.7|17467108|af
|Syria|74.9|72.5|77.4|17185170|me
|Netherlands|81.3|79.2|83.6|17016967|eu
|Ecuador|76.8|73.8|79.9|16080778|sa
|Cambodia|64.5|62|67.1|15957223|ea
|Zambia|52.5|50.8|54.1|15510711|af
|Guatemala|72.3|70.3|74.4|15189958|la
|Zimbabwe|58|57.3|58.7|14546961|af
|Senegal|61.7|59.7|63.8|14320055|af
|Rwanda|60.1|58.5|61.7|12988423|af
|Guinea|60.6|59|62.2|12093349|af
-1|Chad|50.2|49|51.5|11852462|af
|Belgium|81|78.4|83.7|11409077|eu
|Cuba|78.7|76.4|81.1|11179995|la
|Tunisia|76.1|74|78.4|11134588|af
|Burundi|60.5|58.8|62.3|11099298|af
|Bolivia|69.2|66.4|72.1|10969649|sa
|Portugal|79.3|76.1|82.8|10833816|eu
|Somalia|52.4|50.3|54.5|10817354|af
|Greece|80.5|77.9|83.3|10773253|eu
|Benin|61.9|60.5|63.3|10741458|af
|Czechia|78.6|75.7|81.8|10644842|eu
|Dominican Republic|78.1|75.9|80.5|10606865|la
|Haiti|63.8|61.2|66.4|10485800|la
|Sweden|82.1|80.2|84.1|9880604|eu
|Hungary|75.9|72.2|79.8|9874784|eu
|Azerbaijan|72.5|69.5|75.8|9872765|me
-17|Belarus|72.7|67.2|78.6|9570376|eu
|Honduras|71.1|69.5|72.8|8893259|la
|Austria|81.5|78.9|84.3|8711770|eu
|Tajikistan|67.7|64.6|71|8330946|ca
|Jordan|74.6|73.2|76.1|8185384|me
|Switzerland|82.6|80.3|85|8179294|eu
|Israel|82.4|80.6|84.4|8174527|me
|Togo|65|62.3|67.7|7756937|af
|Hong Kong|82.9|80.3|85.8|7167403|ea
|Bulgaria|74.5|71.2|78|7144653|eu
|Serbia|75.5|72.6|78.5|7143921|eu
|Laos|64.3|62.2|66.4|7019073|ea
|Paraguay|77.2|74.5|80|6862812|sa
|Papua New Guinea|67.2|65|69.5|6791317|ea
|Libya|76.5|74.7|78.3|6541948|af
|Lebanon|77.6|76.3|78.9|6237738|me
|El Salvador|74.7|71.4|78.1|6156670|la
|Sierra Leone|58.2|55.6|60.9|6018888|af
|Nicaragua|73.2|71.1|75.5|5966798|la
|United Arab Emirates|77.5|74.8|80.2|5927482|me
|Eritrea|64.9|62.4|67.5|5869869|af
10|Singapore|85|82.3|87.8|5781728|ea
|Kyrgyzstan|70.7|66.5|75.1|5727553|ca
|Denmark|79.4|77|82|5593785|eu
|Central African Republic|52.3|51|53.7|5507257|af
|Finland|80.9|77.9|84|5498211|eu
|Slovakia|77.1|73.5|80.9|5445802|eu
|Turkmenistan|70.1|67.1|73.3|5291317|ca
|Norway|81.8|79.8|83.9|5265158|eu
|Ireland|80.8|78.5|83.2|4952473|eu
|Georgia|76.2|72.1|80.6|4928052|me
|Costa Rica|78.6|75.9|81.4|4872543|la
|Congo, Republic of the|59.3|58.1|60.6|4852412|af
|New Zealand|81.2|79.1|83.3|4474549|oc
|Croatia|75.9|72.7|79.2|4313707|eu
|Liberia|59|57.3|60.8|4299944|af
|Bosnia and Herzegovina|76.7|73.7|80|3861912|eu
|Panama|78.6|75.8|81.6|3705246|la
|Mauritania|63|60.7|65.4|3677293|af
|Puerto Rico|79.4|75.8|83.1|3578056|la
|Moldova|70.7|66.9|74.8|3510485|eu
|Oman|75.5|73.5|77.5|3355262|me
|Uruguay|77.2|74.1|80.5|3351016|sa
|Armenia|74.6|71.4|78.3|3051250|me
|Albania|78.3|75.7|81.2|3038594|eu
|Mongolia|69.6|65.4|74.1|3031330|ea
|Jamaica|73.6|72|75.3|2970340|la
|Lithuania|74.9|69.5|80.6|2854235|eu
|Kuwait|78|76.6|79.4|2832776|me
|West Bank|75|73|77.1|2697687|me
|Namibia|63.6|62.1|65.1|2436469|af
|Qatar|78.7|76.7|80.8|2258283|me
1|Botswana|54.5|56.3|52.6|2209208|af
|Macedonia|76.2|73.6|79|2100025|eu
|Gambia, The|64.9|62.5|67.3|2009648|af
|Slovenia|78.2|74.6|82|1978029|eu
|Latvia|74.5|69.9|79.3|1965686|eu
|Lesotho|53|52.9|53.1|1953070|af
-2|Guinea-Bissau|50.6|48.6|52.7|1759159|af
|Gaza Strip|73.9|72.3|75.7|1753327|me
|Gabon|52.1|51.6|52.5|1738541|af
1|Swaziland|51.6|52.2|51|1451428|af
|Bahrain|78.9|76.7|81.1|1378904|me
|Mauritius|75.6|72.2|79.2|1348242|af
|Timor-Leste|68.1|66.5|69.7|1261072|ea
|Estonia|76.7|71.9|81.7|1258545|eu
|Trinidad and Tobago|72.9|69.9|75.9|1220479|la
|Cyprus|78.7|75.8|81.6|1205575|eu
|Fiji|72.7|70|75.5|915303|oc
|Djibouti|63.2|60.7|65.8|846687|af
|Comoros|64.2|61.9|66.6|794678|af
|Equatorial Guinea|64.2|63.1|65.4|759451|af
|Bhutan|70.1|69.1|71.1|750125|as
|Guyana|68.4|65.4|71.5|735909|sa
|Solomon Islands|75.3|72.7|78.1|635027|oc
-10|Macau|84.5|81.6|87.6|597425|ea
|Western Sahara|63|60.7|65.4|587020|af
|Suriname|72.2|69.8|74.8|585824|sa
|Luxembourg|82.3|79.8|84.9|582291|eu
|Cabo Verde|72.1|69.8|74.5|553432|af
|Brunei|77.2|74.8|79.6|436620|ea
|Malta|80.4|78|82.8|415196|eu
|Maldives|75.6|73.3|78|392960|as
|Belize|68.7|67.2|70.4|353858|la
|Iceland|83|80.9|85.3|335878|eu
|Bahamas, The|72.4|70|74.8|327316|la
|Barbados|75.3|73|77.7|291495|la
|French Polynesia|77.2|74.9|79.6|285321|oc
|Vanuatu|73.4|71.8|75.1|277554|oc
|New Caledonia|77.7|73.7|81.9|275355|oc
|Samoa|73.7|70.8|76.8|198926|oc
|Sao Tome and Principe|64.9|63.6|66.3|197541|af
|Saint Lucia|77.8|75|80.7|164464|la
|Guam|79.1|76.1|82.4|162742|oc
|Curacao|78.3|76|80.7|149035|la
|Aruba|76.8|73.7|79.9|113648|la
|Grenada|74.3|71.7|77.1|111219|la
|Kiribati|66.2|63.7|68.8|106925|oc
|Tonga|76.2|74.7|77.8|106513|oc
|Micronesia, Federated States of|72.9|70.8|75|104719|oc
|Virgin Islands|80|77|83.2|102951|la
|Saint Vincent and the Grenadines|75.3|73.3|77.4|102350|la
|Jersey|81.9|79.4|84.5|98069|eu
|Antigua and Barbuda|76.5|74.4|78.8|93581|la
|Seychelles|74.7|70.2|79.4|93186|af
|Isle of Man|81.2|79.5|83|88195|eu
|Andorra|82.8|80.6|85.1|85660|eu
|Dominica|77|74|80.1|73757|la
|Marshall Islands|73.1|70.9|75.4|73376|oc
|Bermuda|81.3|78.1|84.5|70537|na
|Guernsey|82.5|79.9|85.4|66297|eu
|Greenland|72.4|69.7|75.2|57728|na
|Cayman Islands|81.2|78.5|84|57268|la
|American Samoa|75.4|72.4|78.5|54194|oc
|Northern Mariana Islands|78|75.3|80.8|53467|oc
|Saint Kitts and Nevis|75.7|73.3|78.2|52329|la
|Turks and Caicos Islands|79.8|77.1|82.7|51430|la
|Faroe Islands|80.4|77.8|83.1|50456|eu
|Sint Maarten|78.1|75.8|80.6|41486|la
|Liechtenstein|81.9|79.7|84.6|37937|eu
|British Virgin Islands|78.6|77.2|80.1|34232|la
|San Marino|83.3|80.7|86.1|33285|eu
-1|Monaco|89.5|85.6|93.5|30581|eu
|Gibraltar|79.4|76.6|82.5|29328|eu
|Palau|73.1|69.9|76.5|21347|oc
|Anguilla|81.4|78.8|84.1|16752|la
|Wallis and Futuna|79.7|76.7|82.8|15664|oc
|Tuvalu|66.5|64.3|68.8|10959|oc
|Nauru|67.1|63|70.5|9591|oc
|Cook Islands|75.8|73|78.8|9556|oc
|Saint Helena, Ascension, and Tristan da Cunha|79.5|76.6|82.6|7795|af
|Saint Pierre and Miquelon|80.5|78.2|83|5595|na
1|Montserrat|74.4|75.8|72.9|5267|la
|Falkland Islands (Islas Malvinas)|77.9|75.6|79.6|2931|sa
|Svalbard|NA|NA|NA|2667|eu
|Norfolk Island|NA|NA|NA|2210|oc
|Christmas Island|NA|NA|NA|2205|oc
|Tokelau|NA|NA|NA|1285|oc
|Niue|NA|NA|NA|1190|oc
|Cocos (Keeling) Islands|NA|NA|NA|596|oc
|Pitcairn Islands|NA|NA|NA|54|oc
'''.strip().split('\n')]

# do_refresh_cache = True

def read_url(url, headers={}, path_cache=None, is_verbose=True):
 if (path_cache is None):
  file_cache = os.path.basename(url)
  path_cache = os.path.join('%s.cache' % (os.path.splitext(__file__)[0]),
                            file_cache if (len(file_cache) > 0) else
                            '%s.htm' % (os.path.basename(url.rstrip('/'))))
 if (('do_refresh_cache' in globals() and do_refresh_cache) or
     (not os.path.isfile(path_cache))):
  request = urllib2.Request(url, headers=headers)
  try:                           html = urllib2.urlopen(request).read()
  except urllib2.HTTPError as e: html = ''; print(e)
  try:            os.makedirs(os.path.dirname(path_cache))
  except OSError: pass
  with open(path_cache, 'wb') as f_html: f_html.write(html)
  if (is_verbose): print('%s > %s' % (url, path_cache))
  time.sleep(1) ## avoid rate-limit-exceeded error
 else:
  with open(path_cache) as f_html: html = f_html.read()
  if (is_verbose): print('< %s' % (path_cache))
 try:                       html = html.decode('utf-8')
 except UnicodeDecodeError: pass
 return html
def fmt(string): ## string.format(**vars()) using tags {expression!format} by CMG Lee
 def f(tag): i_sep = tag.rfind('!'); return (re.sub('\.0+$', '', str(eval(tag[1:-1])))
  if (i_sep < 0) else ('{:%s}' % tag[i_sep + 1:-1]).format(eval(tag[1:i_sep])))
 return (re.sub(r'(?<!{){[^{}]+}', lambda m:f(m.group()), string)
         .replace('{{', '{').replace('}}', '}'))
def append(obj, string): return obj.append(fmt(string))
def format_tab(*arg): return '\t'.join([str(el) for el in (arg if len(arg) > 1 else arg[0])])
def tabbify(cellss, separator='|'):
 cellpadss = [list(rows) + [''] * (len(max(cellss, key=len)) - len(rows)) for rows in cellss]
 fmts = ['%%%ds' % (max([len(str(cell)) for cell in cols])) for cols in zip(*cellpadss)]
 return '\n'.join([separator.join(fmts) % tuple(rows) for rows in cellpadss])
def hex_rgb(colour): ## convert [#]RGB to #RRGGBB and [#]RRGGBB to #RRGGBB
 return '#%s' % (colour if len(colour) > 4 else ''.join([c * 2 for c in colour])).lstrip('#')
def try_int_float(field):
 try:     return int(field)
 except:
  try:    return float(field)
  except: return field
def roundm(x, multiple=1):
 try: x[0]; return [roundm(element, multiple) for element in x] ## x[0] checks if x is iterable
 except:    return int(math.floor(float(x) / multiple + 0.5)) * multiple

def findall(regex, string):
 return re.findall(regex, string, flags=re.I|re.DOTALL)
def sub(regex_replace, regex_with, string):
 return str(re.sub(regex_replace, regex_with, string, flags=re.DOTALL).strip())

def make_serial(name): return sub(r'[^a-z]', '', name.lower())
def make_table(datass):
 return '\n'.join(['|'.join([str(data) for data in datas]) for datas in datass])

data_newss = {}

html_expectancy = read_url('http://cia.gov/library/publications/resources/the-world-factbook/fields/355.html')
html_expectancyss = findall(r'(<td.+?</td>)\s*(<td.+?</td>)', html_expectancy)
for html_expectancys in html_expectancyss:
 html_divs = findall(r'<div.+?</div>', html_expectancys[1])
 name      = sub(r'<.*?>', '', html_expectancys[0])
 serial    = make_serial(name)
 # expectancy_male   = None
 # expectancy_female = None
 # try:              expectancy_male   = float(findall(r'[\d.]+(?= years)', html_divs[1])[0])
 # except Exception: pass
 # try:              expectancy_female = float(findall(r'[\d.]+(?= years)', html_divs[2])[0])
 # except Exception: pass
 # if (not serial in data_newss): data_newss[serial] = {}
 # data_newss[serial]['male'  ] = expectancy_male
 # data_newss[serial]['female'] = expectancy_female
 try:
  expectancy_overall = float(findall(r'(?:[\d.]+(?= years)|\d+\.\d+)', html_divs[0])[0])
  expectancy_male    = float(findall(r'(?:[\d.]+(?= years)|\d+\.\d+)', html_divs[1])[0])
  expectancy_female  = float(findall(r'(?:[\d.]+(?= years)|\d+\.\d+)', html_divs[2])[0])
  if (not serial in data_newss): data_newss[serial] = {}
  data_newss[serial]['overall'] = expectancy_overall
  data_newss[serial]['male'   ] = expectancy_male
  data_newss[serial]['female' ] = expectancy_female
 except Exception: pass

html_population = read_url('http://cia.gov/library/publications/resources/the-world-factbook/fields/335.html')
html_populationss = findall(r'(<td.+?</td>)\s*(<td.+?</td>)', html_population)
for html_populations in html_populationss:
 name   = sub(r'<.*?>', '', html_populations[0])
 serial = make_serial(name)
 # population = None
 # if (not 'no indigenous' in html_populations[1]):
 #  try:              population = int(sub(',','',findall(r'[\d,]+', html_populations[1])[0]))
 #  except Exception: pass
 # if (not serial in data_newss): data_newss[serial] = {}
 # data_newss[serial]['population'] = population
 if (not 'no indigenous' in html_populations[1]):
  try:
   population = int(sub(',','',findall(r'[\d,]+', html_populations[1])[0]))
   if (not serial in data_newss): data_newss[serial] = {}
   data_newss[serial]['name']       = name
   data_newss[serial]['population'] = population
  except Exception: pass

outss = []
for serial in sorted(data_newss):
 data_news = data_newss[serial]
 try: outss.append([serial, data_news['name'], data_news['population'],
                    data_news['overall'], data_news['male'], data_news['female']])
                    # data_news['population'] if ('population' in data_news) else None,
                    # data_news['male']       if ('male'       in data_news) else None,
                    # data_news['female']     if ('female'     in data_news) else None])
 except Exception: pass
  # print(data_newss.pop(serial))
# print(tabbify(outss))

outss = []
# print(tabbify(data_oldss))
map_keeps   = {'usa':'unitedstates', 'uk':'unitedkingdom', 'drcongo':'congodemocraticrepublicofthe'}
map_changes = {'swaziland':'eswatini'}

for data_olds in data_oldss:
 name      = data_olds[1]
 serial    = make_serial(name)
 data_news = None
 try:
  if (serial in map_keeps): serial = map_keeps[serial]
  if (serial in map_changes):
   serial    = map_changes[serial]
   data_news = data_newss[serial]
   name      = data_news['name']
  else:
   data_news = data_newss[serial]
 except Exception: pass
 outss.append([data_olds[0],
               name,
               # data_news['name'      ] if ('name'       in data_news) else 'NA',
               data_news['overall'   ] if ('overall'    in data_news) else 'NA',
               data_news['male'      ] if ('male'       in data_news) else 'NA',
               data_news['female'    ] if ('female'     in data_news) else 'NA',
               data_news['population'] if ('population' in data_news) else 'NA',
               data_olds[6]])
 # outss.append(data_olds)
 if (name != data_news['name']): print(name, data_news['name'])
# print(tabbify(outss))
outss = outss[:2] + sorted(outss[2:], key=lambda lines:lines[5], reverse=True)

dir_cache = '%s.cache' % (os.path.splitext(__file__)[0])
with open(os.path.join(dir_cache, 'old.txt'), 'w') as f: f.write(make_table(data_oldss))
with open(os.path.join(dir_cache, 'new.txt'), 'w') as f: f.write(make_table(outss))