mbirth/php-owntracks-recorder
Archived
1
0
Fork 0
Code Issues 1 Pull Requests Activity

Add tool to import data from Google Location History. Optimise database.

Browse Source
This commit is contained in:
Markus Birth 2018-04-27 01:56:54 +02:00
parent b4eb9b42c7
commit b3bc5e199c
Signed by: mbirth
GPG Key ID: A9928D7A098C3A9A
10 changed files with 276 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
/assets/*.js.map
/bower_components/
/tools/Location History.json
/vendor/
config.inc.php
owntracks.db3

3
composer.json
View File

@ -3,7 +3,8 @@
"description": "A simple and responsive self-hosted solution to record and map Owntracks http payloads.",
"require": {
"php": ">=5.6.0",
"mustache/mustache": "~2.12"
"mustache/mustache": "~2.12",
"pcrov/jsonreader": "*"
},
"require-dev": {
},

94
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "ef0251db942fb3eff74bb77aa230b4d2",
"content-hash": "9391fa331565b6c4104fa9908cbc57ea",
"packages": [
{
"name": "mustache/mustache",
@ -51,6 +51,98 @@
"templating"
],
"time": "2017-07-11T12:54:05+00:00"
},
{
"name": "pcrov/jsonreader",
"version": "0.7.1",
"source": {
"type": "git",
"url": "https://github.com/pcrov/JsonReader.git",
"reference": "8baeee5c5c33fe514f63c9d1a06dfe98e5be2501"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pcrov/JsonReader/zipball/8baeee5c5c33fe514f63c9d1a06dfe98e5be2501",
"reference": "8baeee5c5c33fe514f63c9d1a06dfe98e5be2501",
"shasum": ""
},
"require": {
"ext-intl": "*",
"pcrov/unicode": "^0.1",
"php": "^7.0"
},
"require-dev": {
"nst/jsontestsuite": "^1",
"phpunit/phpunit": "^6.5"
},
"type": "library",
"autoload": {
"psr-4": {
"pcrov\\JsonReader\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Paul Crovella",
"role": "developer"
}
],
"description": "JSON Pull Parser",
"homepage": "https://github.com/pcrov/jsonreader",
"keywords": [
"json",
"parser",
"pull",
"streaming"
],
"time": "2018-03-13T01:07:37+00:00"
},
{
"name": "pcrov/unicode",
"version": "0.1.0",
"source": {
"type": "git",
"url": "https://github.com/pcrov/Unicode.git",
"reference": "d2d9d433294ea3b41fd17ff29b4b47fe6f71d83e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pcrov/Unicode/zipball/d2d9d433294ea3b41fd17ff29b4b47fe6f71d83e",
"reference": "d2d9d433294ea3b41fd17ff29b4b47fe6f71d83e",
"shasum": ""
},
"require": {
"php": "^7.0"
},
"require-dev": {
"phpunit/phpunit": "^6.0.8"
},
"type": "library",
"autoload": {
"files": [
"src/functions.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Paul Crovella"
}
],
"description": "Miscellaneous Unicode utility functions",
"homepage": "https://github.com/pcrov/unicode",
"keywords": [
"unicode",
"utf-8"
],
"time": "2018-03-01T09:26:35+00:00"
}
],
"packages-dev": [],

10
lib/Database/AbstractDatabase.php
View File

@ -17,6 +17,16 @@ class AbstractDatabase
// Run query and fetch results
}
public function beginTransaction()
{
// Start transaction
}
public function commitTransaction()
{
// Commit transaction
}
public function isEpochExisting(string $trackerId, int $epoch): bool
{
$sql = 'SELECT epoch FROM ' . $this->prefix . 'locations WHERE tracker_id = ? AND epoch = ?';

10
lib/Database/MySql.php
View File

@ -73,4 +73,14 @@ class MySql extends AbstractDatabase
}
return $result;
}
public function beginTransaction()
{
$this->db->begin_transaction();
}
public function commitTransaction()
{
$this->db->commit();
}
}

10
lib/Database/SQLite.php
View File

@ -42,4 +42,14 @@ class SQLite extends AbstractDatabase
}
return $result;
}
public function beginTransaction()
{
$this->db->beginTransaction();
}
public function commitTransaction()
{
$this->db->commit();
}
}

4
sql/schema_mysql.sql
View File

@ -19,4 +19,6 @@ CREATE TABLE `locations` (
`place_id` int(11) DEFAULT NULL,
`osm_id` int(11) DEFAULT NULL,
`display_name` text
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT;
) DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT;
CREATE INDEX `idx_getmarkers` ON `locations` (`epoch` DESC, `accuracy`, `altitude`);

14
sql/schema_sqlite.sql
View File

@ -8,12 +8,12 @@ CREATE TABLE "locations" (
"heading" INTEGER,
"description" TEXT,
"event" TEXT,
"latitude" REAL,
"longitude" REAL,
"latitude" REAL NOT NULL,
"longitude" REAL NOT NULL,
"radius" INTEGER,
"trig" INTEGER,
"tracker_id" TEXT,
"epoch" INTEGER,
"tracker_id" TEXT NOT NULL,
"epoch" INTEGER NOT NULL,
"vertical_accuracy" INTEGER,
"velocity" INTEGER,
"pressure" REAL,
@ -22,3 +22,9 @@ CREATE TABLE "locations" (
"osm_id" INTEGER,
"display_name" TEXT
);
CREATE INDEX "idx_getmarkers" ON "locations" (
"epoch" DESC,
"accuracy",
"altitude"
);

18
tools/README.md Normal file
View File

@ -0,0 +1,18 @@
Google Location History Import
==============================
This allows you to import your Google Location History (incl. old Google Latitude) data into the
database.
1. Go to [Google Takeout](https://takeout.google.com/).
2. Disable all options by clicking on "SELECT NONE".
3. Find the entry "Location History" and enable it. Make sure the format is set to "JSON".
4. Go to the bottom and click "NEXT". Follow the steps to the end.
5. Wait until you get the completion notification and download the file.
6. Unpack the file and find the `Location History.json`.
7. Put the `Location History.json` in this directory.
8. Modify the `import_google_history.php` and set your desired tracker id in line 5.
9. Run:
```
php import_google_history.php
```

119
tools/import_google_history.php Executable file
View File

@ -0,0 +1,119 @@
#!/usr/bin/env php
<?php
$INPUT_FILE = 'Location History.json'; // file to import (Google Takeout)
$TRACKER_ID = 'mb'; // tracker id to upload the data to
require_once '../config.inc.php';
require_once '../vendor/autoload.php';
use \pcrov\JsonReader\JsonReader;
use \pcrov\JsonReader\InputStream\Stream;
use \OwntracksRecorder\Database\MySql;
use \OwntracksRecorder\Database\SQLite;
$fs = filesize($INPUT_FILE);
$fp = fopen($INPUT_FILE, 'rb');
if ($fp === false) {
throw new IOException(\sprintf("Failed to open URI: %s", $uri));
}
if ($_config['sql_type'] == 'mysql') {
$sql = new MySql($_config['sql_db'], $_config['sql_host'], $_config['sql_user'], $_config['sql_pass'], $_config['sql_prefix']);
} elseif ($_config['sql_type'] == 'sqlite') {
$sql = new SQLite('../' . $_config['sql_db']);
} else {
die('Invalid database type: ' . $_config['sql_type']);
}
$reader = new JsonReader();
$reader->stream($fp);
$reader->read('locations'); // find main structure
$depth = $reader->depth(); // remember depth (only go deeper)
$reader->read(); // goto first element
$ANSI_SCP = chr(0x1b) . '[s';
$ANSI_RCP = chr(0x1b) . '[u';
$ANSI_EL = chr(0x1b) . '[K';
echo 'Importing ' . $INPUT_FILE . ': ' . $ANSI_SCP;
$time_start = microtime(true);
$sql->beginTransaction();
$i = 0;
do {
$data = $reader->value();
#print_r($data);
$accuracy = null;
$altitude = null;
$battery_level = null;
$heading = null;
$description = null;
$event = null;
$latitude = null;
$longitude = null;
$radius = null;
$trig = null;
$tracker_id = null;
$epoch = null;
$vertical_accuracy = null;
$velocity = null;
$pressure = null;
$connection = 'i'; // i = imported
if (array_key_exists('accuracy', $data)) $accuracy = intval($data['accuracy']);
if (array_key_exists('altitude', $data)) $altitude = intval($data['altitude']);
if (array_key_exists('heading', $data)) $heading = intval($data['heading']);
$latitude = floatval($data['latitudeE7']) / 1e7;
$longitude = floatval($data['longitudeE7']) / 1e7;
$epoch = (int)floor(intval($data['timestampMs']) / 1000);
if (array_key_exists('verticalAccuracy', $data)) $vertical_accuracy = intval($data['verticalAccuracy']);
if (array_key_exists('velocity', $data)) $velocity = intval(floatval($data['velocity'])*3.6); // metres per second to km/h
$sql->addLocation(
$accuracy,
$altitude,
$battery_level,
$heading,
$description,
$event,
$latitude,
$longitude,
$radius,
$trig,
$TRACKER_ID,
$epoch,
$vertical_accuracy,
$velocity,
$pressure,
$connection
);
$i++;
if ($i%2000 == 0) {
$sql->commitTransaction();
$sql->beginTransaction();
$pos = ftell($fp);
$frac = $pos/$fs;
$perc = $pos*100/$fs;
$time_now = microtime(true);
$time_spent = $time_now - $time_start;
$time_total = $time_spent / $frac;
$time_left = $time_total - $time_spent;
$time_unit = 'seconds';
if ($time_left > 90) {
$time_left /= 60;
$time_unit = 'minutes';
}
echo $ANSI_RCP . $ANSI_EL . sprintf('%6.2f%% - %.0f %s left', $perc, $time_left, $time_unit);
}
} while ($reader->next() && $reader->depth() > $depth);
$sql->commitTransaction();
echo $ANSI_RCP . $ANSI_EL . 'Done.' . PHP_EOL;
$reader->close();