From 934aaec8471e72fb6a10190e817d3e58b5917c0c Mon Sep 17 00:00:00 2001 From: Jan-Piet Mens Date: Sun, 18 Feb 2024 13:11:31 +0100 Subject: [PATCH] NEW: cached geo records can be auto-expired by setting OTR_CLEAN_AGE fixes #447 --- README.md | 3 ++ recorder.c | 82 +++++++++++++++++++++++++++++++++++++++--------------- udata.h | 1 + 3 files changed, 64 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5f42206..6d42540 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,7 @@ The following configuration variables can be used to tweak settings. Defaults ar | `OTR_PSK` | Y | | MQTT PSK | `OTR_SERVERLABEL` | Y | `OwnTracks` | server label for Web | `OTR_LMDBSIZE` | Y | `5368709120` | size of the LMDB database (5GB). If less than 10485760 (10 MB) it will be set to 10485760. +| `OTR_CLEAN_AGE` | Y | `0` | purge geo gcache entries after these seconds; default 0, disable with 0 ## Reverse proxy @@ -740,6 +741,8 @@ The element `tzname` is the name of the time zone (`$TZ`) at the location. This ``` 4. For historical records which don't have a cached `tzname`, we use the TZDATADB as described above to obtain `tzname`, incurring a not quite insignificant penalty in terms of runtime. +Records in the geo cache have a timestamp (`tst`) which indicates when a particular record was added. As address information and even time zones can become stale (imagine a street you live on getting renamed or a [timezone being renamed](https://github.com/tzinfo/tzinfo/issues/149)) you might wish to expire cached entries. The Recorder does this automatically when configuring `OTR_CLEAN_AGE` to a value of seconds for which entries in the geo cache older than those seconds will be purged if a new reverse-geo lookup succeeds. The default value is 0 which means no cleaning is performed. + ## Lua hooks You can customize Recorder's behavior with Lua hooks. See [HOOKS.md](https://github.com/owntracks/recorder/blob/master/doc/HOOKS.md). diff --git a/recorder.c b/recorder.c index f226fc8..ea22209 100644 --- a/recorder.c +++ b/recorder.c @@ -763,7 +763,8 @@ void handle_message(void *userdata, char *topic, char *payload, size_t payloadle long tst; struct udata *ud = (struct udata *)userdata; char *topics[42]; - int count = 0, cached; + int count = 0; + bool cached, fresh; static UT_string *basetopic = NULL, *username = NULL, *device = NULL, *addr = NULL, *cc = NULL, *ghash = NULL, *ts = NULL; static UT_string *reltopic = NULL, *filename = NULL; char *jsonstring, *_typestr = NULL; @@ -1112,7 +1113,7 @@ void handle_message(void *userdata, char *topic, char *payload, size_t payloadle } - cached = FALSE; + cached = fresh = false; if (ud->revgeo == TRUE) { #ifdef WITH_LUA char *lua_func = "otr_revgeo"; @@ -1133,9 +1134,23 @@ void handle_message(void *userdata, char *topic, char *payload, size_t payloadle } else { #endif /* WITH_LUA */ if ((geo = gcache_json_get(ud->gc, UB(ghash))) != NULL) { - /* Habemus cached data */ + long cache_tst = 0L; - cached = TRUE; + /* We have cached data. See if it's still 'fresh' + * and re-obtain if not. + */ + + fprintf(stderr, "---> CACHED\n"); + if ((j = json_find_member(geo, "tst")) != NULL) { + if (j->tag == JSON_NUMBER) { + cache_tst = j->number_; + } + + if ((time(0) - cache_tst) <= ud->clean_age) { + fresh = true; + } + } + cached = true; if ((j = json_find_member(geo, "cc")) != NULL) { utstring_printf(cc, "%s", j->string_); @@ -1143,22 +1158,39 @@ void handle_message(void *userdata, char *topic, char *payload, size_t payloadle if ((j = json_find_member(geo, "addr")) != NULL) { utstring_printf(addr, "%s", j->string_); } - } else { - if (geoprec > 0) { - if ((geo = revgeo(ud, lat, lon, addr, cc)) != NULL) { - gcache_json_put(ud->gc, UB(ghash), geo); - } else { - /* We didn't obtain reverse Geo, maybe because of over - * quota; make a note of the missing geohash */ + } + if (fresh == false && geoprec > 0) { + static UT_string *taddr = NULL, *tcc = NULL; - char gfile[BUFSIZ]; - FILE *fp; + utstring_renew(taddr); + utstring_renew(tcc); + if ((geo = revgeo(ud, lat, lon, taddr, tcc)) != NULL) { + /* + * We've been able to obtain revgeo; if we + * had old cached data, delete it and add + * new to cache. + */ - snprintf(gfile, BUFSIZ, "%s/ghash/missing", STORAGEDIR); - if ((fp = fopen(gfile, "a")) != NULL) { - fprintf(fp, "%s %lf %lf\n", UB(ghash), lat, lon); - fclose(fp); - } + if (cached) { + gcache_del(ud->gc, UB(ghash)); + } + gcache_json_put(ud->gc, UB(ghash), geo); + + utstring_renew(addr); + utstring_printf(addr, "%s", UB(taddr)); + utstring_renew(cc); + utstring_printf(cc, "%s", UB(tcc)); + } else { + /* We didn't obtain reverse Geo, maybe because of over + * quota; make a note of the missing geohash */ + + char gfile[BUFSIZ]; + FILE *fp; + + snprintf(gfile, BUFSIZ, "%s/ghash/missing", STORAGEDIR); + if ((fp = fopen(gfile, "a")) != NULL) { + fprintf(fp, "%s %lf %lf\n", UB(ghash), lat, lon); + fclose(fp); } } } @@ -1423,6 +1455,7 @@ void usage(char *prog) printf(" --norec don't maintain REC files\n"); printf(" --geokey optional reverse-geo API key\n"); printf(" --debug additional debugging\n"); + printf(" --json-variables -J print settings in JSON and exit\n"); printf(" --variables -V show settings and exit\n"); printf("\n"); @@ -1451,7 +1484,7 @@ int main(int argc, char **argv) int loop_timeout = 1000; #endif int ch, flags, initialize = FALSE; - bool show_variables = false; + bool show_variables = false, show_json_variables = false; static struct udata udata, *ud = &udata; #ifdef WITH_HTTP char *doc_root = DOCROOT; @@ -1512,6 +1545,7 @@ int main(int argc, char **argv) udata.label = strdup("OwnTracks"); udata.geokey = NULL; /* default: no API key */ udata.debug = FALSE; + udata.clean_age = 0L; /* default: don't clean */ flags = LOG_PID; if (isatty(0) || (getenv("DOCKER_RUNNING") != NULL)) { @@ -1569,11 +1603,12 @@ int main(int argc, char **argv) { "viewsdir", required_argument, 0, 16}, #endif { "variables", no_argument, 0, 'V'}, + { "json-variables", no_argument, 0, 'J'}, {0, 0, 0, 0} }; int optindex = 0; - ch = getopt_long(argc, argv, "hDGRi:P:q:S:H:p:A:V", long_options, &optindex); + ch = getopt_long(argc, argv, "hDGRi:P:q:S:H:p:A:JV", long_options, &optindex); if (ch == -1) break; @@ -1677,6 +1712,9 @@ int main(int argc, char **argv) case 'S': strcpy(STORAGEDIR, optarg); break; + case 'J': + show_json_variables = true; + break; case 'V': show_variables = true; break; @@ -1689,8 +1727,8 @@ int main(int argc, char **argv) } - if (show_variables) { - display_variables(ud); + if (show_variables || show_json_variables) { + display_json_variables(ud, show_json_variables ? 0 : 1); exit(0); } diff --git a/udata.h b/udata.h index 43770d3..ea19536 100644 --- a/udata.h +++ b/udata.h @@ -59,6 +59,7 @@ struct udata { int debug; /* enable for debugging */ struct gcache *httpfriends; /* lmdb named database 'friends' */ struct gcache *wpdb; /* lmdb named database 'wp' (waypoints) */ + long clean_age; /* how long in seconds to keep geo gcache entries */ }; #endif