Move files into own table. Fix double UTF-8 encoding.

lib/TclUpdates/SQLiteWriter.php

@@ -16,17 +16,87 @@ class SQLiteWriter
         }
     }
 
-    private function insertArray($table, $data)
+    private function insertArray($table, $data, $replace = false)
     {
         $placeholders = array_fill(0, count($data), '?');
-        $sql = 'INSERT INTO "' . $table . '" (' . implode(', ', array_keys($data)) . ') VALUES (' . implode(', ', $placeholders) . ')';
+        $sql = 'INSERT ';
+        if ($replace) {
+            $sql .= 'OR REPLACE ';
+        }
+        $sql .= 'INTO "' . $table . '" (' . implode(', ', array_keys($data)) . ') VALUES (' . implode(', ', $placeholders) . ')';
         $stmt = $this->pdo->prepare($sql);
         $ok = $stmt->execute(array_values($data));
         return $ok;
     }
 
+    public function addFile($file_arr)
+    {
+        // Try fetch previous entry
+        $sql = 'SELECT * FROM "files" WHERE "sha1"=?';
+        $stmt = $this->pdo->prepare($sql);
+        $ok = $stmt->execute(array($file_arr['file_sha1']));
+        $result = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+        $pubFirst = '2099-12-31';
+        $pubLast  = '1970-01-01';
+        $note = array('en' => null, 'ja' => null, 'zh' => null);
+        if (count($result) > 0) {
+            $pubFirst = $result[0]['published_first'];
+            $pubLast  = $result[0]['published_last'];
+            $note = json_decode($result[0]['note'], true);
+            $hasChanged = false;
+        } else {
+            $hasChanged = true;
+        }
+
+        if (strtotime($file_arr['pubDate']) < strtotime($pubFirst)) {
+            $pubFirst = $file_arr['pubDate'];
+            $hasChanged = true;
+        }
+
+        if (strtotime($file_arr['pubDate']) > strtotime($pubLast)) {
+            $pubLast = $file_arr['pubDate'];
+            $hasChanged = true;
+        }
+
+        foreach ($file_arr['note'] as $lang => $desc) {
+            // TODO: Maybe improve, i.e. compare for different contents
+            if (strpos($desc, chr(0xc3)) !== false) {
+                // fix double-UTF-8 encoding
+                $desc = utf8_decode($desc);
+            }
+            if (mb_strlen($desc) > mb_strlen($note[$lang])) {
+                $note[$lang] = $desc;
+                $hasChanged = true;
+            }
+        }
+
+        if ($hasChanged) {
+            $this->insertArray('files', array(
+                'sha1' => $file_arr['file_sha1'],
+                'file_name' => $file_arr['file_name'],
+                'file_size' => $file_arr['file_size'],
+                'type' => $file_arr['type'],
+                'note' => json_encode($note, JSON_UNESCAPED_UNICODE),
+                'published_first' => $pubFirst,
+                'published_last' => $pubLast,
+            ), true);
+        }
+    }
+
     public function addGotu(GotuObject $g)
     {
+        $this->addFile(array(
+            'file_sha1' => $g->file_chksum,
+            'file_name' => $g->filename,
+            'file_size' => $g->file_size,
+            'type' => $g->type,
+            'note' => array(
+                'en' => $g->description_en,
+                'ja' => $g->description_ja,
+                'zh' => $g->description_zh,
+            ),
+            'pubDate' => $g->time,
+        ));
         $ok = $this->insertArray('updates', array(
             'tv' => $g->tv,
             'fv' => $g->fv,
@@ -35,15 +105,7 @@ class SQLiteWriter
             'publisher' => $g->publisher,
             'fwId' => $g->fw_id,
             'file_id' => $g->file_id,
-            'file_name' => $g->filename,
-            'file_size' => $g->file_size,
             'file_sha1' => $g->file_chksum,
-            'type' => $g->type,
-            'note' => json_encode(array(
-                'en' => $g->description_en,
-                'ja' => $g->description_ja,
-                'zh' => $g->description_zh,
-            ))
         ));
         if ($ok) {
             $key = $this->pdo->lastInsertId();

sql/dbschema.sql

@@ -38,6 +38,16 @@ INSERT INTO "devices" ("ref", "modelId", "name") VALUES
   ("PRD-63116-001", 1, "Unlocked USA")
 ;
 
+CREATE TABLE "files" (
+  "sha1" TEXT UNIQUE PRIMARY KEY,   -- checksum of file
+  "file_name" TEXT,      -- filename of file
+  "file_size" INTEGER,   -- size
+  "type" TEXT,           -- FULL(4) or OTA(2) update
+  "note" TEXT,           -- description of file (optional)
+  "published_first" INTEGER,   -- stamp of earliest pubdate
+  "published_last" INTEGER     -- stamp of latest pubdate
+);
+
 -- we only care about the first file for now
 -- a separate "files" table might get introduced later
 CREATE TABLE "updates" (
@@ -49,11 +59,7 @@ CREATE TABLE "updates" (
   "publisher" TEXT,    -- publisher
   "fwId" TEXT,     -- <FW_ID> (CHANGES FOR THE SAME FILE_ID!!!) MAYBE MOVE TO update_map
   "file_id" TEXT,      -- <FILE_ID> of first file
-  "file_name" TEXT,    -- filename of first file
+  "file_sha1" TEXT     -- SHA1 checksum of first file
-  "file_size" INTEGER, -- size of first file
-  "file_sha1" TEXT,    -- SHA1 checksum of first file
-  "type" TEXT,     -- FULL or OTA
-  "note" TEXT      -- some note for this file (optional)
 );
 CREATE UNIQUE INDEX "index_updates" ON "updates" (
   "tv",