From 1297b31b2ea7c1e73373563906a561024cd12acd Mon Sep 17 00:00:00 2001 From: Markus Birth Date: Thu, 8 Feb 2018 01:15:58 +0100 Subject: [PATCH] Initial rewrite. --- tclcheck_allota.py | 31 ++++----- tcllib/requests/__init__.py | 5 ++ tcllib/requests/http.py | 38 +++++++++++ tcllib/requests/runner.py | 48 ++++++++++++++ tcllib/requests/serverselector.py | 105 ++++++++++++++++++++++++++++++ tcllib/requests/tcl.py | 96 +++++++++++++++++++++++++++ tcllib/requests/tclresult.py | 21 ++++++ 7 files changed, 329 insertions(+), 15 deletions(-) create mode 100644 tcllib/requests/__init__.py create mode 100644 tcllib/requests/http.py create mode 100644 tcllib/requests/runner.py create mode 100644 tcllib/requests/serverselector.py create mode 100644 tcllib/requests/tcl.py create mode 100644 tcllib/requests/tclresult.py diff --git a/tclcheck_allota.py b/tclcheck_allota.py index a219fca..70705d6 100644 --- a/tclcheck_allota.py +++ b/tclcheck_allota.py @@ -7,16 +7,14 @@ import sys -from requests.exceptions import RequestException - import tcllib import tcllib.argparser from tcllib import ansi, devlist from tcllib.devices import MobileDevice +from tcllib.requests import RequestRunner, CheckRequest, ServerVoteSelector dev = MobileDevice() -fc = tcllib.FotaCheck() dpdesc = """ Checks for the latest OTA updates for all PRD numbers or only for the PRD specified @@ -39,23 +37,26 @@ prds = devlist.get_devicelist() print("List of latest OTA firmware{} by PRD:".format(force_ver_text)) +runner = RequestRunner(ServerVoteSelector()) +runner.max_tries = 20 + for prd, variant in prds.items(): model = variant["variant"] lastver = variant["last_ota"] lastver = variant["last_full"] if lastver is None else lastver if args.forcever is not None: lastver = args.forcever - if prdcheck in prd: - try: - dev.curef = prd - dev.fwver = lastver - fc.reset_session(dev) - check_xml = fc.do_check(dev, max_tries=20) - curef, fv, tv, fw_id, fileid, fn, fsize, fhash = fc.parse_check(check_xml) - versioninfo = ansi.YELLOW_DARK + fv + ansi.RESET + " ⇨ " + ansi.YELLOW + tv + ansi.RESET + " (FULL: {})".format(variant["last_full"]) - print("{}: {} {} ({})".format(prd, versioninfo, fhash, model)) - except RequestException as e: - print("{} ({}): {}".format(prd, lastver, str(e))) - continue + if not prdcheck in prd: + continue + dev.curef = prd + dev.fwver = lastver + chk = CheckRequest(dev) + runner.run(chk) + if chk.success: + result = chk.get_result() + versioninfo = ansi.YELLOW_DARK + result.fvver + ansi.RESET + " ⇨ " + ansi.YELLOW + result.tvver + ansi.RESET + " (FULL: {})".format(variant["last_full"]) + print("{}: {} {} ({})".format(prd, versioninfo, result.filehash, model)) + else: + print("{} ({}): {}".format(prd, lastver, chk.error)) tcllib.FotaCheck.write_info_if_dumps_found() diff --git a/tcllib/requests/__init__.py b/tcllib/requests/__init__.py new file mode 100644 index 0000000..c06739f --- /dev/null +++ b/tcllib/requests/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from .tcl import * +from .runner import * +from .serverselector import * diff --git a/tcllib/requests/http.py b/tcllib/requests/http.py new file mode 100644 index 0000000..2b2821a --- /dev/null +++ b/tcllib/requests/http.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +import requests +from collections import OrderedDict + +class TimeoutException(Exception): + pass + +class HttpRequest: + """Provides all generic features for making HTTP GET requests""" + def __init__(self, url, timeout=10): + self.url = url + self.params = OrderedDict() + self.timeout = timeout + self.headers = {} + + def reset_session(self): + """Reset everything to default.""" + self.sess = requests.Session() + self.sess.headers.update(self.headers) + + def run(self): + """Run query.""" + try: + req = self.sess.get(self.url, params=self.params, timeout=self.timeout) + except requests.exceptions.Timeout as e: + raise TimeoutException(e) + return req + +class HttpPostRequest(HttpRequest): + """Provides all generic features for making HTTP POST requests""" + def run(self): + """Run query.""" + try: + req = self.sess.post(self.url, data=self.params, timeout=self.timeout) + except requests.exceptions.Timeout as e: + raise TimeoutException(e) + return req diff --git a/tcllib/requests/runner.py b/tcllib/requests/runner.py new file mode 100644 index 0000000..8f812e2 --- /dev/null +++ b/tcllib/requests/runner.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +from . import tcl +from . import http +from . import serverselector + +class UnknownMethodException(Exception): + pass + +class RequestRunner: + def __init__(self, server_selector: serverselector.ServerSelector, https=True): + self.server_selector = server_selector + self.protocol = "https://" if https else "http://" + self.max_tries = 5 + + def get_http(self, method="GET") -> http.HttpRequest: + """Returns the http class according to desired method.""" + if method == "GET": + return http.HttpRequest + elif method == "POST": + return http.HttpPostRequest + raise UnknownMethodException("Unknown http method: {}".format(method)) + + def get_server(self) -> str: + """Returns a master server.""" + return self.server_selector.get_master_server() + + def run(self, query: tcl.TclRequest, timeout: int=10) -> bool: + """Runs the actual query.""" + for _ in range(0, self.max_tries): + url = "{}{}{}".format(self.protocol, self.get_server(), query.uri) + http_handler = self.get_http(query.method)(url, timeout) + http_handler.headers = query.get_headers() + http_handler.params = query.get_params() + http_handler.reset_session() + self.server_selector.hook_prerequest() + try: + req = http_handler.run() + req.encoding = "utf-8" + done = query.is_done(req.status_code, req.text) + self.server_selector.hook_postrequest(done) + if done: + return done + except http.TimeoutException: + self.server_selector.hook_postrequest(False) + query.error = "Timeout." + query.error = "Max tries ({}) reached.".format(self.max_tries) + return False diff --git a/tcllib/requests/serverselector.py b/tcllib/requests/serverselector.py new file mode 100644 index 0000000..2877353 --- /dev/null +++ b/tcllib/requests/serverselector.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +# pylint: disable=C0111,C0326,C0103 + +"""Tools to sort API servers to find the least awful one.""" + +import numpy +import time + + +MASTER_SERVERS = [ + "g2master-us-east.tclclouds.com", + "g2master-us-west.tclclouds.com", + "g2master-eu-west.tclclouds.com", + "g2master-ap-south.tclclouds.com", + "g2master-ap-north.tclclouds.com", + "g2master-sa-east.tclclouds.com", +] + +class ServerSelector: + """Returns a random server to use.""" + + def __init__(self): + """Init stuff""" + self.last_server = None + + def get_master_server(self): + """Return a random server.""" + while True: + new_server = numpy.random.choice(MASTER_SERVERS) + if new_server != self.last_server: + break + self.last_server = new_server + return new_server + + def hook_prerequest(self): + """Hook to be called before doing request""" + pass + + def hook_postrequest(self, successful: bool): + """Hook to be called after request finished""" + pass + +class ServerVoteSelector(ServerSelector): + """Tries to return faster servers more often.""" + + def __init__(self): + """Populate server list and weighting variables.""" + self.last_server = None + self.master_servers_weights = [3] * len(MASTER_SERVERS) + self.check_time_sum = 3 + self.check_time_count = 1 + + def get_master_server(self): + """Return weighted choice from server list.""" + weight_sum = 0 + for i in self.master_servers_weights: + weight_sum += i + numpy_weights = [] + for i in self.master_servers_weights: + numpy_weights.append(i/weight_sum) + self.last_server = numpy.random.choice(MASTER_SERVERS, p=numpy_weights) + return self.last_server + + def master_server_downvote(self): + """Decrease weight of last chosen server.""" + idx = MASTER_SERVERS.index(self.last_server) + if self.master_servers_weights[idx] > 1: + self.master_servers_weights[idx] -= 1 + + def master_server_upvote(self): + """Increase weight of last chosen server.""" + idx = MASTER_SERVERS.index(self.last_server) + if self.master_servers_weights[idx] < 10: + self.master_servers_weights[idx] += 1 + + def check_time_add(self, duration): + """Record connection time.""" + self.check_time_sum += duration + self.check_time_count += 1 + + def check_time_avg(self): + """Return average connection time.""" + return self.check_time_sum / self.check_time_count + + def master_server_vote_on_time(self, last_duration): + """Change weight of a server based on average connection time.""" + avg_duration = self.check_time_avg() + if last_duration < avg_duration - 0.5: + self.master_server_upvote() + elif last_duration > avg_duration + 0.5: + self.master_server_downvote() + + def hook_prerequest(self): + """Hook to be called before doing request""" + self.reqtime_start = time.perf_counter() + + def hook_postrequest(self, successful: bool): + """Hook to be called after request finished""" + reqtime = time.perf_counter() - self.reqtime_start + self.check_time_add(reqtime) + if successful: + self.master_server_vote_on_time(reqtime) + else: + self.master_server_downvote() diff --git a/tcllib/requests/tcl.py b/tcllib/requests/tcl.py new file mode 100644 index 0000000..8e08ac1 --- /dev/null +++ b/tcllib/requests/tcl.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +from .. import devices +from . import tclresult +from collections import OrderedDict +from defusedxml import ElementTree + +class TclRequest: + def __init__(self): + self.uri = "" + self.result = None + self.error = None + self.success = False + + def get_headers(self): + return {} + + def get_params(self): + return {} + + def is_done(self, http_status: int, contents: str): + """Checks if query is done or needs retry.""" + return False + + def get_result(self): + """Returns Result object.""" + return None + +class CheckRequest(TclRequest): + def __init__(self, device: devices.Device): + super().__init__() + self.uri = "/check.php" + self.method = "GET" + self.device = device + + def get_headers(self): + return {"User-Agent": self.device.ua} + + def get_params(self): + params = OrderedDict() + params["id"] = self.device.imei + params["curef"] = self.device.curef + params["fv"] = self.device.fwver + params["mode"] = self.device.mode + params["type"] = self.device.type + params["cltp"] = self.device.cltp + params["cktp"] = self.device.cktp + params["rtd"] = self.device.rtd + params["chnl"] = self.device.chnl + #params["osvs"] = self.device.osvs + #params["ckot"] = self.device.ckot + return params + + def is_done(self, http_status: int, contents: str) -> bool: + ok_states = { + 204: "No update available.", + 404: "No data for requested CUREF/FV combination.", + } + if http_status == 200: + self.result = contents + self.success = True + return True + elif http_status in ok_states: + self.error = ok_states[http_status] + self.success = False + return True + elif http_status not in [500, 502, 503]: + # Errors OTHER than 500, 502 or 503 are probably + # errors where we don't need to retry + self.error ="HTTP {}.".format(http_status) + self.success = False + return True + return False + + def get_result(self): + if not self.success: + return None + return tclresult.CheckResult(self.result) + +# Check requests have 4 possible outcomes: +# 1. HTTP 200 with XML data - our desired info +# 2. HTTP 204 - means: no newer update available +# 3. HTTP 404 - means: invalid device or firmware version +# 4. anything else: server problem (esp. 500, 502, 503) + + + +class DownloadRequest(TclRequest): + def __init__(self, device: devices.Device): + super().__init__() + self.uri = "/download_request.php" + self.method = "POST" + self.device = device + + def get_headers(self): + return {"User-Agent": self.device.ua} diff --git a/tcllib/requests/tclresult.py b/tcllib/requests/tclresult.py new file mode 100644 index 0000000..e801956 --- /dev/null +++ b/tcllib/requests/tclresult.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from defusedxml import ElementTree + + +class TclResult: + pass + +class CheckResult(TclResult): + def __init__(self, xml: str): + self.raw_xml = xml + root = ElementTree.fromstring(xml) + self.curef = root.find("CUREF").text + self.fvver = root.find("VERSION").find("FV").text + self.tvver = root.find("VERSION").find("TV").text + self.fw_id = root.find("FIRMWARE").find("FW_ID").text + fileinfo = root.find("FIRMWARE").find("FILESET").find("FILE") + self.fileid = fileinfo.find("FILE_ID").text + self.filename = fileinfo.find("FILENAME").text + self.filesize = fileinfo.find("SIZE").text + self.filehash = fileinfo.find("CHECKSUM").text