#!/usr/bin/env python3 # -*- coding: utf-8 -*- import configparser import datetime import fnmatch import os import sys import xml.etree.ElementTree as ET import MosDb import MosXml c = configparser.ConfigParser() c.read("config.ini") today_str = "{:%Y%m%d}".format(datetime.datetime.now()) MOS_LOG_PATH = c.get("MOS", "XML_LOG_PATH") # TODO: Store and read from database or ini! LAST_PARSED_FILE = "MOSMessage-20170221-120000.xml" if os.path.isfile(".last_file"): with open(".last_file", "r") as f: LAST_PARSED_FILE = f.readline().strip() f.close() # Scan available MOSMessage logs and collect those we still need to scan files_to_parse = [] for file in os.listdir(MOS_LOG_PATH): if fnmatch.fnmatch(file, "MOSMessage-*.xml"): # Only use less-than to re-read the file we had last (might have add. data) if file < LAST_PARSED_FILE: continue files_to_parse.append(MOS_LOG_PATH + file) num_files = len(files_to_parse) print("Found {} files to be parsed.".format(num_files)) #sys.exit(0) db = MosDb.MosDb() file_count = 0 last_file = "" for file in files_to_parse: file_count += 1 last_file = os.path.basename(file) tree = ET.parse(file) print("\n{} [{:d}/{:d}]: ".format(file, file_count, num_files), end="") root = tree.getroot() # Walk all objects for elem in root: # Skip everything which is not a mosObj message if elem.attrib.get("command") != "mosObj": continue mos = elem.find("mos") objData = MosXml.parse_mos_xml(mos) if objData['isDeleted'] == 0: db.add_or_replace(objData) print(".", end="") else: db.mark_deleted(objData) print("x", end="") db.finish() print("") print("Last file: {}".format(last_file)) with open(".last_file", "w") as f: f.write(last_file) f.close()