#!/usr/bin/env python3 # -*- coding: utf-8 -*- #-- pip3 install -U fiona #--- pip3 install -U csvtomd """ Write new wikidata labels ... """ import argparse import csv import logging import sys import re from collections import defaultdict import fiona logging.basicConfig(stream=sys.stderr, level=logging.INFO) parser = argparse.ArgumentParser(description='Update wikidata labels for Natural-Earth ') parser.add_argument('-input_shape', default='../../10m_cultural/ne_10m_populated_places.shp', help='input natural-earth shape file - with wikidataid columns') parser.add_argument('-input_lettercase', default='lowercase', help='variables in the shape file - lowercase or uppercase') parser.add_argument('-input_csv', default='ne_10m_populated_places.csv', help='input csv file with the latest wikidataid name_XX columns') parser.add_argument('-output_shape', default='ne_10m_populated_places_latest.shp', help='output natural-earth shape file - with updated wikidataid name_XX columns') parser.add_argument('-output_csvlog', default='ne_10m_populated_places_latest.log.csv', help='audit log about the changes') parser.add_argument('-output_csvsumlog', default='ne_10m_populated_places_latest.sumlog.csv', help='audit sum log about the changes') args = parser.parse_args() riverclean_regex = re.compile(r'\b('+'River|rivière de la |rivière à la|rivière des|Rivière De|Rivière du|Rivière aux|Rivière|rivier|Rio dos|Rio|Río La|Río de los|Río de las|Río dos|Río|sông|-folyó|folyó|canale di|canale|Nehri|Jiang|Sungai'+r')\b', flags=re.IGNORECASE) # Some of these are proper names (Lake of the Ozark's, Clear Lake Reservoir) and # shouldn't be stripped, but in the meantime, strip aggressively lakeclean_regex = re.compile(r'\b('+'Lake of the|Grand Lake o\' the|Lake Reservoir|Grand Lake|Grant Lake|Lake of|Lake|Lago degli|Lago del|Lago de|lago di|Lago la|Lago do|Lago |lago d\'||Lago|Lac de la|lac d\'|lac des|Lac des|lac de|Lac de|Lac au|lac di|lac la|Lac La|Lac à l’|lac à la|Lac|lac|-See|See|Laguna de|Laguna|Lake Reservoir|Reservoir|réservoir de la|réservoir de|Reservatório de|réservoir|Réservoir|Represa de|Represa|baie de|Bahía de|öböl|Gölü|järv|Embalse de|Embalse|Bacino di|bacino di|Bacino|bacino|Sông|Lough|Hồ|Danau'+r')\b', flags=re.IGNORECASE) #geolabels_regex = re.compile(r'\b('+'(wyspa)'+r')\b', # flags=re.IGNORECASE) placeclean_regex = re.compile(r'\b('+'Municipality of|Municipality|First Nation|Urban District|urban area|Distretto di|District de|District|Heroica Ciudad de|Município|Contea di|Barra do|City of'+r')\b', flags=re.IGNORECASE) geo_region_regex = re.compile(r'\b('+'Região Autónoma dos'+r')\b', flags=re.IGNORECASE) admin2_regex = re.compile(r'\b('+'County|Condado de|comté de|contea di|comté d|megye|Hrabstwo|Condado de|Quận|مقاطعة|City and Borough|census area di|Census Area|Borough|borough di|borough de|ilçesi'+r')\b', flags=re.IGNORECASE) admin1_regex = re.compile(r'\b('+'canton of |Canton of|Department|District Council|distretto di contea di|contea di|District de|distretto di|Distretto della|Distrik|distretto del|district|Constitutional Province|Province of |Provincia del|provincia delle|provincia della|provincia di|Provincia Constitucional del|Província Constitucional de|provincia de|Província de|Província do|Provincia de|Préfecture de|Provincia|Comunidad De|Autonome Provinz|Provincia Autónoma de|Provinz|Província|Departamento de|Departamento do|Autonomous Province of|Autonomous Province|Province de la|province de|Province du|Province|Provinsi|Municipality of |Municipality|Município de|Special Region of |Región Metropolitana de|Special Region|Autonomous Region|Capital Region of|Region of|-Region|Region|Governorate|Gouvernorat|Gubernatorstwo|Gobernación de|governatorato di|Capital of|Capital of|City Council|City and Borough of|City of|Città di|City|Región Metropolitana de|Metropolitan Region|Metropolitan Borough of|Metropolitan Borough|Borough Metropolitano de|London Borough of|district londonien|district londonien d\'|borough royal de|Royal Borough of|County Borough|Borough of|Borough Council|Metropoliten Borough|londonien de|district royal de|County|Old Royal Capital|(distrikt)|Distrik|(borough)|Cantão central de|cantone della|(cantão)|(departamento)|(departement)|Región del|Región de|Región|gouvernorat de|Gouvernorat|kormányzóság|regione di|Regione del|Prefectura de|prefettura di|Autonome Oblast|Oblast Autônomo|Autonomous Oblast|Oblast\' dell\'|Obwód Autonomiczny|Kraï de|Kraï|Oblast de|Óblast de|Oblast\' di|Oblast\'|oblast|distrito de|Distrito do|Distrito|métropolitain de|Voivodia da|cantone di|cantone dell\'|Munisipalitas\' di|Munisipalitas|Emirato di|Emirato|cantón del|cantón de|canton du|cantón|Καντόνι του|Καντόνι της|Ζουπανία του|Επαρχία του|Δήμος|Κυβερνείο του|distretto|Région autonome du|région de|Governamento de|Kegubernuran|comté de|parrocchia di|obwód|, London|, Londra|, Nya Zeeland|Daerah Istimewa|Autónoma del|Parish of|Parish|, Barbados|Circondario autonomo dei|Circondario autonomo|circondario autonomo degli|Okręg Autonomiczny|Dystrykt|-Distrikt|Distrikt|distriktet|, प्रांत|, पैराग्वे|, Zambia|, Kenya|, Καμερούν|, Τζαμάικα|, Barbados|, Londra|, Bahama|kommun|Ciudad de|, Gambia|, Botswana|tartomány|körzet|Munizip|division|Conselho do Borough de|Rejon|Raionul|Kotar|megye|Żupania|comune distrettuale di|comune distrettuale|Comune di|comune|Condado de|Condado|Kotamadya|Região Autónoma dos|Região Autónoma|Região|Guvernementet|Gobernación del|Gobernación'+r')\b', flags=re.IGNORECASE) admin0_regex = re.compile(r'\b('+'(district)|(địa hạt)'+r')\b', flags=re.IGNORECASE) def isNotEmpty(s): """ Check String is not empty """ return bool(s and s.strip()) # read csv data wddic = defaultdict(dict) wdredirects = defaultdict(dict) name_field_prefix = 'name_' languages = ['ar','bn','de','el','en','es','fa','fr','he','hi','hu','id','it','ja','ko','nl','pl','pt','ru','sv','tr','uk','ur','vi','zh','zht'] new_properties = [] with open(args.input_csv, newline='') as csvfile: reader = csv.DictReader(csvfile) for row in reader: qid = row['wd_id'] for d in row: if d != 'wd_id': #print(d, d[0:5]) if d[0:5] == 'name_': wddic[qid][d] = row[d].strip() # Clean names ... # where input_shape evaluates the name of the shapefile # as proxy for featureclass # Rivers ... if args.input_shape.lower().find('rivers_lake') > 0: wddic[qid][d] = riverclean_regex.sub('', wddic[qid][d]) # Comma ... if wddic[qid][d].find(',') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('(')] # Lakes ... if args.input_shape.lower().find('lake') > 0 and args.input_shape.lower().find('10m_physical') > 0: #if d == 'name_en' and wddic[qid]['name_en'] != 'Lake of the Woods': wddic[qid][d]=lakeclean_regex.sub('', wddic[qid][d] ) # Comma ... if wddic[qid][d].find(',') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('(')] # Physical Geography label polys and points ... if args.input_shape.lower().find('_geography_') > 0: wddic[qid][d]=geo_region_regex.sub('', wddic[qid][d] ) # Comma ... if wddic[qid][d].find(',') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('(')] # Places ... if args.input_shape.lower().find('place') > 0: wddic[qid][d] = placeclean_regex.sub('', wddic[qid][d]) #name_zh: remove last "市""City" character if d == 'name_zh' and wddic[qid]['name_zh'] and wddic[qid]['name_zh'][-1] == "市": wddic[qid]['name_zh'] = wddic[qid]['name_zh'][:-1] # Comma ... if wddic[qid][d].find(',') > 0: # RTL languages wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find(',')] # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('(')] # Admin 2 counties ... if args.input_shape.lower().find('admin_2') > 0: wddic[qid][d] = admin2_regex.sub('', wddic[qid][d]) # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('(')] #name_ko: remove last "주""State" character if d == 'name_ko' and wddic[qid]['name_ko'] and wddic[qid]['name_ko'][-1] == "군": wddic[qid]['name_ko'] = wddic[qid]['name_ko'][:-1] # Admin 1 states, provinces ... if args.input_shape.lower().find('admin_1') > 0: wddic[qid][d] = admin1_regex.sub('', wddic[qid][d]) # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('(')] #name_ko: remove last "주""State" character if d == 'name_ko' and wddic[qid]['name_ko'] and wddic[qid]['name_ko'][-1] == "주": wddic[qid]['name_ko'] = wddic[qid]['name_ko'][:-1] # That's the city name, we're only interested in the region name if wddic[qid][d] == 'Washington, D.C.': wddic[qid][d] = 'D.C.' # Admin 0 countries, map units, etc if args.input_shape.lower().find('admin_0') > 0: wddic[qid][d] = admin0_regex.sub('', wddic[qid][d]) # Parenthetical ... if wddic[qid][d].find('(') > 0: # RTL languages and LTR figure each other out in python 3 wddic[qid][d] = wddic[qid][d][0:wddic[qid][d].find('(')] # sometimes the RegEx strips words mid string and leaves double spaces # we also want to strip leading or trailing whitespace wddic[qid][d] = wddic[qid][d].replace(' ', ' ').strip() if wddic[qid][d] != row[d].strip(): print(qid, d, ' name cleaning : ', row[d].strip(), ' --> ', wddic[qid][d]) # Add an uppercase version if args.input_lettercase != "lowercase": wddic[qid][d.upper()] = wddic[qid][d] # - add uppercase version del wddic[qid][d] # - remove default lowcase if row['wd_id_new'] != '': wdredirects[qid] = row['wd_id_new'] with open(args.output_csvlog, "w", encoding='utf-8') as f: writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) writer.writerow(("wd_id", "status", "variable", "value_old", "value_new")) with fiona.open(args.input_shape, 'r', encoding='utf-8', driver="ESRI Shapefile") as source: out_schema = source.schema for locale in languages: name_locale = name_field_prefix + locale if args.input_lettercase != "lowercase": name_locale = name_locale.upper() if name_locale not in out_schema['properties']: print("...NOTE.. adding locale to schema", name_locale) out_schema['properties'][name_locale] = 'str' new_properties.append(name_locale) #print(out_schema) # **source.meta is a shortcut to get the crs, driver, and schema #with fiona.open(args.output_shape, 'w', encoding='utf-8', **source.meta) as sink: # But since we're overwriting the schema, we take the long way # keyword arguments from the source Collection. with fiona.open( args.output_shape, 'w', driver=source.driver, crs=source.crs, encoding='utf-8', schema=out_schema, ) as sink: #print(sink.schema) warn_no_dbf_field_for_new_property = 1 stat_equal = 0 stat_empty = 0 stat_new = 0 stat_new_value_new_lang = 0 stat_del = 0 stat_mod = 0 stat_wikidataid_redirect = 0 stat_wikidataid_notfound = 0 stat_wikidataid_null = 0 stat_wikidataid_badformated = 0 stat_wikidataid_notnull = 0 for f in source: try: # get in format at least for new_prop in new_properties: f['properties'][new_prop] = '' if args.input_lettercase == "lowercase": qid = f['properties']['wikidataid'] else: qid = f['properties']['WIKIDATAID'] if not qid: qid = '' if qid.startswith('Q'): stat_wikidataid_notnull += 1 if qid in wddic: #Update wikidataid if redirects if qid in wdredirects: writer.writerow((qid, "REDIRECT", 'wikidataid', qid, wdredirects[qid])) stat_wikidataid_redirect += 1 if args.input_lettercase == "lowercase": f['properties']['wikidataid'] = wdredirects[qid] else: f['properties']['WIKIDATAID'] = wdredirects[qid] for updatefield in wddic[qid]: try: if not f['properties'][updatefield]: f['properties'][updatefield] = '' else: f['properties'][updatefield] = f['properties'][updatefield].strip() except: # visual logging is handled in the next section # see warn_no_dbf_field_for_new_property pass if not wddic[qid][updatefield]: wddic[qid][updatefield] = '' else: wddic[qid][updatefield] = wddic[qid][updatefield].strip() if updatefield in f['properties']: if f['properties'][updatefield] != wddic[qid][updatefield]: if f['properties'][updatefield] == '': if updatefield in new_properties: status = 'NEWvalue_NEW_LANG' stat_new_value_new_lang += 1 else: status = 'NEWvalue' stat_new += 1 elif wddic[qid][updatefield] == '': status = 'DELvalue' stat_del += 1 else: status = 'MODvalue' stat_mod += 1 writer.writerow((qid, status, updatefield, f['properties'][updatefield], wddic[qid][updatefield])) f['properties'][updatefield] = wddic[qid][updatefield] else: if isNotEmpty(f['properties'][updatefield]): stat_equal += 1 else: stat_empty += 1 else: if warn_no_dbf_field_for_new_property: print("...ERROR.. updatefield is missing", updatefield) warn_no_dbf_field_for_new_property = 0 #sys.exit(1) status = 'NEWvalue_NEW_LANG' stat_new_value_new_lang += 1 writer.writerow((qid, status, updatefield, '', wddic[qid][updatefield])) f['properties'][updatefield] = wddic[qid][updatefield] #f['properties'].update(updatefield=wddic[qid][updatefield]) #print(f['properties']) else: print("WARNING: Not found wikidataid - please check :", qid) stat_wikidataid_notfound += 1 else: if qid == '': stat_wikidataid_null += 1 else: stat_wikidataid_badformated += 1 print("WARNING: Bad formatted wikidataid , skip", qid) sink.write(f) except Exception as e: logging.exception("Error processing feature %s:", f['id']) print(e) sys.exit(1) # Write statistics ... with open(args.output_csvsumlog, "w", encoding='utf-8') as s: sumWriter = csv.writer(s, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) sumWriter.writerow(("shapefilename", "var", "value")) sumWriter.writerow((args.input_shape, 'New_name', stat_new)) sumWriter.writerow((args.input_shape, 'New_name_new_language', stat_new_value_new_lang)) sumWriter.writerow((args.input_shape, 'Deleted_name', stat_del)) sumWriter.writerow((args.input_shape, 'Modified_name', stat_mod)) sumWriter.writerow((args.input_shape, 'Empty_name', stat_empty)) sumWriter.writerow((args.input_shape, 'Same_name', stat_equal)) sumWriter.writerow((args.input_shape, 'Wikidataid_redirected', stat_wikidataid_redirect)) sumWriter.writerow((args.input_shape, 'Wikidataid_notfound', stat_wikidataid_notfound)) sumWriter.writerow((args.input_shape, 'Wikidataid_null', stat_wikidataid_null)) sumWriter.writerow((args.input_shape, 'Wikidataid_notnull', stat_wikidataid_notnull)) sumWriter.writerow((args.input_shape, 'Wikidataid_badformated', stat_wikidataid_badformated))