From 30b5d08e34ea1e8f747e57702d44d29d79e74258 Mon Sep 17 00:00:00 2001
From: Markus Birth <mbirth@gmail.com>
Date: Sat, 16 Dec 2017 22:15:12 +0100
Subject: [PATCH] Cleanup tool to remove duplicate XMLs.

---
 clean_dupes.php | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100755 clean_dupes.php

diff --git a/clean_dupes.php b/clean_dupes.php
new file mode 100755
index 0000000..0c280cb
--- /dev/null
+++ b/clean_dupes.php
@@ -0,0 +1,24 @@
+#!/usr/bin/env php
+<?php
+
+$bkup_dir = __DIR__ . '/data/';
+
+$file_list = glob($bkup_dir . '*.xml');
+
+$hashes = array();
+foreach ($file_list as $file) {
+    $filename = basename($file);
+    $file_hash = sha1_file($file);
+
+    if (isset($hashes[$file_hash])) {
+        $old_file = $hashes[$file_hash];
+        if (md5_file($file) == md5_file($bkup_dir . $old_file)) {
+            echo 'Duplicate file: ' . $filename . ' (first: ' . $old_file . ')' . PHP_EOL;
+            unlink($file);
+            continue;
+        }
+        echo 'Possible SHA1 collision?' . PHP_EOL;
+    }
+
+    $hashes[$file_hash] = $filename;
+}