location resolution, functioning, some tests

.gitignore

@@ -1 +1,3 @@
 venv
+data/.cache.csv
+__pycache__

README.md

@@ -3,25 +3,19 @@ Find time now, in the past or future in any timezone or location.
 
 ## Usage
 ```
-python tz.py HUMAN_TIME [HUMAN_TZ_LOC] [--format="%Y-%m-%d %H:%M:%S%z"]
+python tz.py QUERY [--format="%Y-%m-%d %H:%M:%S%z"] [--debug]
 ```
 
-* `HUMAN_TIME` - is any time or time-like string. See the [dateparser](https://pypi.org/project/dateparser/) for example `10:15`, `now`, `in 3 hours` and many others.
+* `QUERY` - is of the form `<datetime-like> to <timezone or location>`
-* `HUMAN_TZ_LOC` - (optional) is either timezone to which the date should be translated (fast) or a location (slow - and requires internet connection to resolve the location). Uses [geopy](https://geopy.readthedocs.io/en/stable/) for location resolution and [timezonefinder](https://pypi.org/project/timezonefinder/) for timezone resolution.
+
+`<datetime-like>` is any time or time-like string - or example `2019-04-26 3:14`, `06:42`, timezones `15:10 cet` but also some human readable like `now`, `in 3 hours`, `7 minutes ago` and many others. See [dateparser](https://pypi.org/project/dateparser/) for more.
+
+`to <timezone or location>` is optional. It has to have the word `to` which specifies that timezone or location follows. It is either:
+  - timezone (tried first) to which the date should be translated or 
+  - a location. Uses a local database of files of countries and cities. It then tries to fuzzymatch the query using [fuzzywuzzy](https://github.com/seatgeek/fuzzywuzzy). In case it can't find the country or city, it uses [geopy](https://geopy.readthedocs.io/en/stable/) for location resolution. Finally it uses [timezonefinder](https://pypi.org/project/timezonefinder/) for timezone resolution.
+
 * `--format` is the format of the time to be displayed. See supported [datetime formats](https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior)
 
-You could alias the whole command to `tww` for faster typing, e.g. in your `.bashrc`:
-
-```
-alias tww="~/workspace/tz/venv/bin/python ~/workspace/tz/tz.py"
-```
-
-If `HUMAN_TIME` and/or `HUMAN_TZ_LOC` have spaces, they need to be quoted, e.g.:
-
-```
-tww "in 2 hours" "los angeles"
-```
-
 ## Install
 
 ```
@@ -30,41 +24,47 @@ source venv/bin/activate
 pip install -r requirements.txt
 ```
 
+You could alias the whole command to `tww` for faster typing, e.g. in your `.bashrc`:
+
+```
+alias tww="~/workspace/tz/venv/bin/python ~/workspace/tz/tz.py"
+```
+
 ## Examples
 
-Without timezone/location:
+Time now (in this timezone):
 
 ```
 $ tww now
-2019-03-13 15:04:36.607080
+2019-03-13 15:04:36
 ```
 
-Time now in another timezone
+Time now to another timezone (UTC let's say):
 ```
-$ tww now CET
+$ tww now to utc
-2019-03-13 23:07:54.957743
+2019-03-13 15:04:36
 ```
 
 One hour from now in UTC, showing only the time:
 ```
-$ tww "in 1 hour" utc --format="%T"
+$ tww in 1 hour to cet --format="%T"
 23:17:49
 ```
 
 With timezone:
 ```
-$ tww now Asia/Tokyo
+$ tww now to asia/tokyo
-2019-03-14 07:06:35.273192
+2019-03-14 07:06:35
 ```
 
-Another time in timezone:
+Another time to timezone:
 ```
-$ tww 15:10 cet
+$ tww 15:10 to cet
 2019-03-13 23:10:00
 ```
 
-Time in location (**slow!**):
+Time in one timezone (pst) to another in city:
 ```
-$ tww "3/14 15 9:26:53 PST" sofia
+$ tww 3/14 15 9:26:53 PST to sofia
 2015-03-14 19:26:53+02:00
 ```

data/countries.csv

@@ -0,0 +1,240 @@
+andorra,42.5,1.5
+united arab emirates,24,54
+afghanistan,33,65
+antigua and barbuda,17.05,-61.8
+anguilla,18.25,-63.17
+albania,41,20
+armenia,40,45
+netherlands antilles,12.25,-68.75
+angola,-12.5,18.5
+asia/pacific region,35,105
+antarctica,-90,0
+argentina,-34,-64
+american samoa,-14.33,-170
+austria,47.33,13.33
+australia,-27,133
+aruba,12.5,-69.97
+azerbaijan,40.5,47.5
+bosnia and herzegovina,44,18
+barbados,13.17,-59.53
+bangladesh,24,90
+belgium,50.83,4
+burkina faso,13,-2
+bulgaria,43,25
+bahrain,26,50.55
+burundi,-3.5,30
+benin,9.5,2.25
+bermuda,32.33,-64.75
+brunei darussalam,4.5,114.67
+bolivia,-17,-65
+brazil,-10,-55
+bahamas,24.25,-76
+bhutan,27.5,90.5
+bouvet island,-54.43,3.4
+botswana,-22,24
+belarus,53,28
+belize,17.25,-88.75
+canada,60,-95
+cocos (keeling) islands,-12.5,96.83
+"congo, the democratic republic of the",0,25
+central african republic,7,21
+congo,-1,15
+switzerland,47,8
+cote d'ivoire,8,-5
+cook islands,-21.23,-159.77
+chile,-30,-71
+cameroon,6,12
+china,35,105
+colombia,4,-72
+costa rica,10,-84
+cuba,21.5,-80
+cape verde,16,-24
+christmas island,-10.5,105.67
+cyprus,35,33
+czech republic,49.75,15.5
+germany,51,9
+djibouti,11.5,43
+denmark,56,10
+dominica,15.42,-61.33
+dominican republic,19,-70.67
+algeria,28,3
+ecuador,-2,-77.5
+estonia,59,26
+egypt,27,30
+western sahara,24.5,-13
+eritrea,15,39
+spain,40,-4
+ethiopia,8,38
+europe,47,8
+finland,64,26
+fiji,-18,175
+falkland islands (malvinas),-51.75,-59
+"micronesia, federated states of",6.92,158.25
+faroe islands,62,-7
+france,46,2
+gabon,-1,11.75
+united kingdom,54,-2
+grenada,12.12,-61.67
+georgia,42,43.5
+french guiana,4,-53
+ghana,8,-2
+gibraltar,36.18,-5.37
+greenland,72,-40
+gambia,13.47,-16.57
+guinea,11,-10
+guadeloupe,16.25,-61.58
+equatorial guinea,2,10
+greece,39,22
+south georgia and the south sandwich islands,-54.5,-37
+guatemala,15.5,-90.25
+guam,13.47,144.78
+guinea-bissau,12,-15
+guyana,5,-59
+hong kong,22.25,114.17
+heard island and mcdonald islands,-53.1,72.52
+honduras,15,-86.5
+croatia,45.17,15.5
+haiti,19,-72.42
+hungary,47,20
+indonesia,-5,120
+ireland,53,-8
+israel,31.5,34.75
+india,20,77
+british indian ocean territory,-6,71.5
+iraq,33,44
+"iran, islamic republic of",32,53
+iceland,65,-18
+italy,42.83,12.83
+jamaica,18.25,-77.5
+jordan,31,36
+japan,36,138
+kenya,1,38
+kyrgyzstan,41,75
+cambodia,13,105
+kiribati,1.42,173
+comoros,-12.17,44.25
+saint kitts and nevis,17.33,-62.75
+"korea, democratic people's republic of",40,127
+"korea, republic of",37,127.5
+kuwait,29.34,47.66
+cayman islands,19.5,-80.5
+kazakhstan,48,68
+lao people's democratic republic,18,105
+lebanon,33.83,35.83
+saint lucia,13.88,-61.13
+liechtenstein,47.17,9.53
+sri lanka,7,81
+liberia,6.5,-9.5
+lesotho,-29.5,28.5
+lithuania,56,24
+luxembourg,49.75,6.17
+latvia,57,25
+libyan arab jamahiriya,25,17
+morocco,32,-5
+monaco,43.73,7.4
+"moldova, republic of",47,29
+montenegro,42,19
+madagascar,-20,47
+marshall islands,9,168
+macedonia,41.83,22
+mali,17,-4
+myanmar,22,98
+mongolia,46,105
+macao,22.17,113.55
+northern mariana islands,15.2,145.75
+martinique,14.67,-61
+mauritania,20,-12
+montserrat,16.75,-62.2
+malta,35.83,14.58
+mauritius,-20.28,57.55
+maldives,3.25,73
+malawi,-13.5,34
+mexico,23,-102
+malaysia,2.5,112.5
+mozambique,-18.25,35
+namibia,-22,17
+new caledonia,-21.5,165.5
+niger,16,8
+norfolk island,-29.03,167.95
+nigeria,10,8
+nicaragua,13,-85
+netherlands,52.5,5.75
+norway,62,10
+nepal,28,84
+nauru,-0.53,166.92
+niue,-19.03,-169.87
+new zealand,-41,174
+oman,21,57
+panama,9,-80
+peru,-10,-76
+french polynesia,-15,-140
+papua new guinea,-6,147
+philippines,13,122
+pakistan,30,70
+poland,52,20
+saint pierre and miquelon,46.83,-56.33
+puerto rico,18.25,-66.5
+palestinian territory,32,35.25
+portugal,39.5,-8
+palau,7.5,134.5
+paraguay,-23,-58
+qatar,25.5,51.25
+reunion,-21.1,55.6
+romania,46,25
+serbia,44,21
+russian federation,60,100
+rwanda,-2,30
+saudi arabia,25,45
+solomon islands,-8,159
+seychelles,-4.58,55.67
+sudan,15,30
+sweden,62,15
+singapore,1.37,103.8
+saint helena,-15.93,-5.7
+slovenia,46,15
+svalbard and jan mayen,78,20
+slovakia,48.67,19.5
+sierra leone,8.5,-11.5
+san marino,43.77,12.42
+senegal,14,-14
+somalia,10,49
+suriname,4,-56
+sao tome and principe,1,7
+el salvador,13.83,-88.92
+syrian arab republic,35,38
+swaziland,-26.5,31.5
+turks and caicos islands,21.75,-71.58
+chad,15,19
+french southern territories,-43,67
+togo,8,1.17
+thailand,15,100
+tajikistan,39,71
+tokelau,-9,-172
+turkmenistan,40,60
+tunisia,34,9
+tonga,-20,-175
+turkey,39,35
+trinidad and tobago,11,-61
+tuvalu,-8,178
+taiwan,23.5,121
+"tanzania, united republic of",-6,35
+ukraine,49,32
+uganda,1,32
+united states minor outlying islands,19.28,166.6
+united states,38,-97
+uruguay,-33,-56
+uzbekistan,41,64
+holy see (vatican city state),41.9,12.45
+saint vincent and the grenadines,13.25,-61.2
+venezuela,8,-66
+"virgin islands, british",18.5,-64.5
+"virgin islands, u.s.",18.33,-64.83
+vietnam,16,106
+vanuatu,-16,167
+wallis and futuna,-13.3,-176.2
+samoa,-13.58,-172.33
+yemen,15,48
+mayotte,-12.83,45.17
+south africa,-29,24
+zambia,-15,30
+zimbabwe,-20,30

requrements.txt

@@ -1,11 +1,29 @@
+backcall==0.1.0
 dateparser==0.7.1
+decorator==4.3.2
+freezegun==0.3.11
+fuzzywuzzy==0.17.0
 geographiclib==1.49
 geopy==1.18.1
 importlib-resources==1.0.2
+ipython==7.3.0
+ipython-genutils==0.2.0
+jedi==0.13.3
 numpy==1.16.2
+parso==0.3.4
+pexpect==4.6.0
+pickleshare==0.7.5
+pkg-resources==0.0.0
+prompt-toolkit==2.0.9
+ptyprocess==0.6.0
+Pygments==2.3.1
 python-dateutil==2.8.0
+python-Levenshtein==0.12.0
 pytz==2018.9
 regex==2019.3.12
 six==1.12.0
 timezonefinder==4.0.1
+traitlets==4.3.2
 tzlocal==1.5.1
+wcwidth==0.1.7
+word2number==1.1

test_tz.py

@@ -0,0 +1,49 @@
+import datetime
+import unittest
+
+from freezegun import freeze_time
+import pytz
+from tz import query_to_format_result, setup_logging_level
+
+def get_local_hours_offset():
+    now = datetime.datetime.now()
+    local_tzinfo = now.astimezone().tzinfo
+    delta = local_tzinfo.utcoffset(now)
+    return delta.total_seconds()/3600
+
+FROZEN_TIME = "2015-03-14 09:26:53 UTC"
+
+# TODO: not quite sure....
+# This gives the UTC+XX.YY offset.
+# In order to get frozen time in UTC, use the negative as offset
+UTC = -get_local_hours_offset()
+
+# LocalTimezone, Query, Result
+INTEGRATION_TESTS = [
+    (UTC, "now", "2015-03-14 09:26:53"),
+    # in two hours -> 11:26 | to tokyo +9 -> 20:26
+    (0, "in two hours to asia/tokyo", "2015-03-14 20:26:53"),
+    # now in UTC +2hrs -> 11:26
+    (0, "now to sofia", "2015-03-14 11:26:53"),
+]
+
+
+class IntegrationTests(unittest.TestCase):
+    def setUp(self):
+        setup_logging_level(True)
+
+    def test_integration(self):
+        for test in INTEGRATION_TESTS:
+            local_tz = test[0]
+            query = test[1].split(" ")
+            result = test[2]
+            
+            freezer = freeze_time(FROZEN_TIME, tz_offset=local_tz)
+            
+            freezer.start()
+            self.assertEqual(query_to_format_result(query), result)
+            freezer.stop()
+
+
+if __name__ == '__main__':
+    unittest.main()

tz.py

@@ -1,50 +1,257 @@
-import sys
+"""
+"""
+
 import argparse
+import logging
+import sys
 
 import dateparser
 from pytz.exceptions import UnknownTimeZoneError
 
+FUZZ_THRESHOLD = 70
+DEFAULT_FORMAT = '%Y-%m-%d %H:%M:%S%z'
 
-parser = argparse.ArgumentParser()
+logging.basicConfig()
-parser.add_argument('human_dt', help="datetime-like string")
+logger = logging.getLogger()
-parser.add_argument('human_tz', nargs='?', help="timezone-like or location string")
-parser.add_argument('--format', dest='format', default='%Y-%m-%d %H:%M:%S%z')
-args = parser.parse_args()
 
-human_dt = args.human_dt
+def parse_args():
-human_tz = args.human_tz
+    parser = argparse.ArgumentParser()
+    parser.add_argument('query', nargs='*', help="<datetime-like> to <timezone-like or location string>")
+    parser.add_argument('--format', dest='format', default=DEFAULT_FORMAT)
+    parser.add_argument('--debug', dest='debug', action='store_true')
+    args = parser.parse_args()
+    return args
 
-try:
+
-    # first try parsing the timezone from user input
+def setup_logging_level(debug=False):
-    if human_tz:
+    log_level = logging.DEBUG if debug else logging.ERROR
-        result = dateparser.parse(human_dt, settings={'TO_TIMEZONE': human_tz})
+    logger.setLevel(log_level)
-    else:
+    logger.debug("Debugging enabled")
-        result = dateparser.parse(human_dt)
+
-except UnknownTimeZoneError:
+
-    # we don't know this timezone one, assume location
+
+class Location:
+    """
+    Represents a location with name, latitude and longitude
+    """
+    def __init__(self, name:str, latitude: float, longitude: float):
+        self.name = name
+        self.latitude = latitude
+        self.longitude = longitude
+    
+    def __lt__(self, other):
+        return self.name < other.name
+    
+    def __str__(self):
+        return "{} {} {}".format(self.name, self.latitude, self.longitude)
+
+
+def normalize_words_to_number(query):
+    """
+    Converts queries like "in one hour" -> "in 1 hour"
+    Assumes one-word numbers used
+    """
+    from word2number import w2n
+    
+    normal_list = []
+    
+    for word in query.split():
+        try:
+            normal_list.append(str(w2n.word_to_num(word)))
+        except ValueError:
+            normal_list.append(word)
+    normal = ' '.join(normal_list)
+    logger.debug("Normalized dt query: {} -> {}".format(query, normal))
+    return normal
+
+
+def timezone_to_normal(query):
+    """
+    Makes a timezone written in wrong capitalization to correct one 
+    as expected by IANA. E.g.:
+        america/new_york -> America/New_York
+    """
+    import re
+    
+    # The magic in the regex is that it splits by either / OR _ OR -
+    # where the | are OR; and then the parens ( ) keep the splitting
+    # entries in the list so that we can join later
+    normal = ''.join(x.capitalize() for x in re.split('(/|_|-)', query))
+    logger.debug("Normalized timezone: {} -> {}".format(query, normal))
+    return normal
+
+
+def create_if_not_exists(fname):
+    try:
+        fh = open(fname, 'r')
+    except FileNotFoundError:
+        fh = open(fname, 'w')
+    fh.close()
+
+
+def write_to_cache(query, location):
+    import os
+    import csv
+    
+    logger.debug("Writing location to cache")
+    with open(os.path.join("data",".cache.csv"), 'a+') as wf:
+        cachewriter = csv.writer(wf)
+        cachewriter.writerow([query,
+                              location.latitude,
+                              location.longitude])
+
+
+def row_to_location(row):
+    """
+    Row from a csv file to location class
+    """
+    latitude, longitude = float(row[1]), float(row[2])
+    return Location(row[0], latitude, longitude)
+
+
+def resolve_location_local(query):
+    """
+    Find a location by searching in local db of countries and cities
+    """
+    import os
+    import csv
+    from heapq import heappush, heappop
+    from fuzzywuzzy import fuzz
+
+    query = query.lower()
+    create_if_not_exists(os.path.join("data",".cache.csv"))
+    
+    # location hypothesis heap
+    heap = []
+    
+    for fname in [".cache", "countries", "cities"]:
+        with open(os.path.join("data", "{}.csv".format(fname))) as f:
+            cfile = csv.reader(f)
+            for row in cfile:
+                entry = row[0]
+                if fname == ".cache" and entry == query:
+                    location = row_to_location(row)
+                    logger.debug("Location (from cache): {}".format(location))
+                    return location
+                fuzz_ratio = fuzz.ratio(query, entry)
+                if fuzz_ratio > FUZZ_THRESHOLD:
+                    location = row_to_location(row)
+                    logger.debug("Location hyp ({} {}): {}".format(fuzz_ratio, fname, location))
+                    # need to push negative result as heapq is min heap
+                    heappush(heap, (-fuzz_ratio, location))
+    try:
+        result = heappop(heap)
+    except IndexError:
+        logger.critical("Could not find location {}".format(query))
+        exit(1)
+    ratio, location = result
+    logger.debug("Location result ({}): {}".format(-ratio, location))
+    write_to_cache(query, location)
+    return location
+
+
+def resolve_location_remote(query):
     import random
     import string
 
     from geopy.geocoders import Nominatim
     from geopy.exc import GeocoderTimedOut
-    from timezonefinder import TimezoneFinder
 
     user_agent = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20))
     geolocator = Nominatim(user_agent=user_agent)
     try:
-        location = geolocator.geocode(human_tz)
+        location = geolocator.geocode(query)
+        write_to_cache(query, location)
+        return location
     except GeocoderTimedOut:
-        print("Timed out resolving location. Try specifying a timezone directly", file=sys.stderr)
+        logger.critical("Timed out resolving location. Try specifying a timezone directly")
         exit(1)
 
 
-    tzf = TimezoneFinder()
+def parse_query(query):
-    loc_tz = tzf.timezone_at(lng=location.longitude, lat=location.latitude)
+    """
-    result = dateparser.parse(human_dt, settings={'TO_TIMEZONE': loc_tz})
+    Parses the user query to the datetime, tz/loc parts
+    """
+    to_query = ' '.join(query).split(" to ")
+    logger.debug("to_query: {}".format(to_query))
+    if len(to_query) == 1:
+        # only datetime
+        human_dt, human_tz_loc = to_query[0], None
+    elif len(to_query) == 2:
+        # datetime to timezone
+        human_dt, human_tz_loc = to_query
+    else:
+        logger.critical("There can be only one 'to' in the query string")
+        exit(1)
 
-if result is None:
+    logger.debug("raw human_dt: {}".format(human_dt))
-    print("Could not parse '{human_dt}' or '{human_tz}'".format(human_dt, human_tz), file=sys.stderr)
+    logger.debug("raw human_tz_loc: {}".format(human_tz_loc))
-    exit(1)
     
-formatted_result = result.strftime(args.format)
+    human_dt = normalize_words_to_number(human_dt)
-print(formatted_result)
+    
+    return human_dt, human_tz_loc
+
+
+def solve_query(human_dt, human_tz_loc):
+    try:
+        # first try parsing the timezone from user input
+        result = dateparser.parse(human_dt)
+        logger.debug("human_dt result: {}".format(result))
+        if human_tz_loc:
+            # if the human_tz_loc contains /, assume it's a timezone which could be
+            # incorrectly written with small letters - need Continent/City
+            if "/" in human_tz_loc:
+                human_tz_loc = timezone_to_normal(human_tz_loc)
+            isofmt = result.isoformat()
+            logger.debug("human_dt isofmt: {}".format(isofmt))
+            result = dateparser.parse(isofmt, settings={'TO_TIMEZONE': human_tz_loc})
+            logger.debug("human_dt to_timezone result: {}".format(result))
+    except UnknownTimeZoneError:
+        from timezonefinder import TimezoneFinder
+        logger.debug("No timezone: {}".format(human_tz_loc))
+        # if the human_tz_loc contains /, assume it's a timezone 
+        # the timezone could still be guessed badly, attempt to get the city
+        # e.g.america/dallas
+        if "/" in human_tz_loc:
+            logger.debug("Assuming wrongly guessed tz {}".format(human_tz_loc))
+            human_tz_loc = human_tz_loc.split('/')[-1]
+            logger.debug("Try city {}".format(human_tz_loc))
+        # we don't know this timezone one, assume location
+        # Try to get from local file first
+        location = resolve_location_local(human_tz_loc)
+        if not location:
+            # finally go to remote
+            location = resolve_location_remote(human_tz_loc)
+        tzf = TimezoneFinder()
+        loc_tz = tzf.timezone_at(lat=location.latitude, lng=location.longitude)
+        logger.debug("Timezone: {}".format(loc_tz))
+        result = dateparser.parse(human_dt, settings={'TO_TIMEZONE': loc_tz})
+    return result
+
+
+def format_result(result, fmt):
+    if result is None:
+        logger.critical("Could not s query")
+        exit(1)
+    logger.debug("Format: {}".format(fmt))
+    format_result = result.strftime(fmt)
+    logger.debug("Formated result: {} -> {}".format(result, format_result))
+    return format_result
+
+
+def query_to_format_result(query, fmt=DEFAULT_FORMAT):
+    human_dt, human_tz_loc = parse_query(query)
+    result = solve_query(human_dt, human_tz_loc)
+    formated_result = format_result(result, fmt)
+    return formated_result
+
+
+def main(args):
+    formated_result = query_to_format_result(args.query, args.format)
+    print(formated_result)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    setup_logging_level(args.debug)
+    main(args)