mbirth/apt-urlcheck
Archived
1
0
Fork 0
Code Issues Pull Requests Activity

Rewrite to Python.

Browse Source
This commit is contained in:
Markus Birth 2018-02-15 00:51:51 +01:00
parent 172dafcf4a
commit bde616fc60
Signed by: mbirth
GPG Key ID: A9928D7A098C3A9A
4 changed files with 148 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
ansi.py Normal file
View File

@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
"""Listing of ANSI colors plus additional Windows support."""
import platform
# Needed to make ANSI escape sequences work in Windows
SYSTEM = platform.system()
if SYSTEM == "Windows":
try:
import colorama
colorama.init()
except ImportError:
pass
UP_DEL = u"\u001b[F\u001b[K"
BLACK = u"\u001b[0;30m"
RED_DARK = u"\u001b[0;31m"
GREEN_DARK = u"\u001b[0;32m"
YELLOW_DARK = u"\u001b[0;33m"
CYAN_DARK = u"\u001b[0;36m"
SILVER = u"\u001b[0;37m"
GREY = u"\u001b[1;30m"
RED = u"\u001b[1;31m"
GREEN = u"\u001b[1;32m"
YELLOW = u"\u001b[1;33m"
CYAN = u"\u001b[1;36m"
WHITE = u"\u001b[1;37m"
RESET = u"\u001b[0m"

279
apt-urlcheck.php
View File

@ -1,279 +0,0 @@
#!/usr/bin/php
<?php
class APTChecker
{
private $whitelist = array();
private $aptlists = array();
private $codenames_old = array( 'gutsy', 'hardy', 'intrepid', 'jaunty', 'karmic', 'lucid', 'maverick', 'natty', 'oneiric', 'precise', 'quantal', 'raring', 'saucy', 'trusty', 'utopic', 'vivid', 'wily' );
private $codenames = array( 'devel', 'xenial', 'yakkety', 'zesty', 'artful', 'bionic', 'debian', 'squeeze', 'stable', 'unstable', 'beta' );
private $codename;
function __construct()
{
$this->codename = exec('lsb_release -cs');
$this->addSourceLists( '/etc/apt/sources.list' );
$this->addSourceLists( '/etc/apt/sources.list.d/*.list' );
}
public function addSourceLists( $filemask )
{
$this->aptlists = array_merge( $this->aptlists, glob( $filemask ) );
}
public function getSourceLists()
{
return $this->aptlists;
}
public function addWhitelist( $codename )
{
if ( !is_array( $codename ) ) $codename = array( $codename );
foreach ( $codename as $cn ) {
$this->whitelist[] = $cn;
}
}
public function isRoot()
{
return (posix_getuid() == 0);
}
public function getCodename()
{
return $this->codename;
}
public function parseLists()
{
$debs = array();
foreach ( $this->getSourceLists() as $path ) {
$fc = file($path, FILE_IGNORE_NEW_LINES|FILE_SKIP_EMPTY_LINES);
foreach ($fc as $i=>$fline) {
if ((substr($fline, 0, 4) == 'deb ') || (substr($fline, 0, 8) == 'deb-src ')) {
$debs[] = array(
'file' => $path,
'line' => $i+1,
'data' => $fline,
);
}
}
}
return $debs;
}
public function parseDebLine( $line )
{
$parts = explode(' ', $line);
if ( $parts[1]{0} == '[' ) {
$result = array(
'type' => $parts[0],
'attributes' => $parts[1],
'url' => $parts[2],
'distr' => $parts[3],
'components' => array_slice($parts, 4),
);
} else {
$result = array(
'type' => $parts[0],
'attributes' => '',
'url' => $parts[1],
'distr' => $parts[2],
'components' => array_slice($parts, 3),
);
}
return $result;
}
private function tryFetch( $url ) {
$answer = array();
stream_context_set_default( array(
'http' => array(
'method' => 'HEAD',
'timeout' => 5.0,
),
) );
$result = get_headers( $url, 1 );
$status_line = $result[0];
if ( is_array( $status_line ) ) {
$status_line = end( $status_line );
}
$status = substr( $status_line, 9, 3 );
stream_context_set_default( array(
'http' => array(
'method' => 'GET',
),
) );
return ( $status == '200' );
}
private function tryGetDirectoryListing( $url )
{
$all_known_codenames = array_merge($this->codenames_old, $this->codenames);
$list = @file_get_contents( $url );
if ( $list === false ) return false;
preg_match_all('/<a .*?href=[\'"]?([^\'"]+)[\'"]?.*?>/i', $list, $matches);
#print_r($matches);
$result = array();
foreach ($matches[1] as $match) {
if ($match{0} != '?' && $match{0} != '/' && substr($match, -1) == '/' && $match != '../' && substr($match, 0, 4) != 'http') {
$result[] = substr($match, 0, -1);
} elseif (in_array($match, $all_known_codenames) || in_array(substr($match, 0, -1), $all_known_codenames)) {
$result[] = $match;
}
}
#print_r($result);
return array_unique( $result );
}
private function tryReleases( $baseurl, $additional = array() )
{
if ( !is_array( $additional ) ) $additional = array( $additional );
$result = array();
foreach ( array_unique( array_merge( $this->codenames, $additional ) ) as $codename ) {
foreach ( array( 'InRelease', 'Release', 'Release.gpg' ) as $filename ) {
$try_url = $baseurl . '/dists/' . $codename . '/' . $filename;
$exists = $this->tryFetch( $try_url );
if ( $exists !== false ) {
$result[] = $codename;
break;
}
}
}
return $result;
}
private function getServerInfo( $info )
{
$result = array();
$result['method'] = 'dirlist';
$dirlist = $this->tryGetDirectoryListing( $info['url'] . '/dists' );
if ( $dirlist === false ) {
// Damn!
$dirlist = $this->tryReleases( $info['url'], $info['distr'] );
$result['method'] = 'bruteforce';
}
$result['dists'] = $dirlist;
return $result;
}
public function analyzeDebs( $debs = null, $progress = null )
{
if (is_null($debs)) $debs = $this->parseLists();
$result = array();
foreach ( $debs as $deb ) {
$info = $this->parseDebLine( $deb['data'] );
if ( substr( $info['distr'], 0, strlen( $this->codename ) ) != $this->codename && !in_array( $info['distr'], $this->whitelist ) ) {
$serverinfo = $this->getServerInfo( $info );
$result[] = array(
'deb' => $deb,
'info' => $info,
'server' => $serverinfo,
);
}
if ($this->isRoot()) $this->getKeyId($info['url'], $info['distr']); // cache key-id
if (!is_null($progress)) call_user_func($progress);
}
return $result;
}
private function getKeyId( $url, $dist )
{
global $keycache;
if (!isset($keycache)) $keycache = array();
if (isset($keycache[$url][$dist])) return $keycache[$url][$dist];
$sigfile = @file_get_contents($url.'/dists/'.$dist.'/Release.gpg');
if ($sigfile === false) {
echo 'No signature found.' . PHP_EOL;
$keycache[$url][$dist] = false;
return false;
}
exec('echo "' . $sigfile . '" | gpg --batch --verify - /etc/passwd 2>&1', $output, $retval);
$output = implode(PHP_EOL, $output);
preg_match('/gpg: Signature made (.*) using (.*) key ID (.*)\r?\n/m', $output, $matches);
$result = array(
'date' => $matches[1],
'type' => $matches[2],
'id' => $matches[3],
'desc' => $matches[4],
);
echo 'Repo uses key id ' . $result['id'] . PHP_EOL;
if (empty($result['id'])) {
echo 'DEBUG: ' . $output . PHP_EOL;
}
$keycache[$url][$dist] = $result;
return $result;
}
public function outputResults( $debinfo ) {
$all_codenames = array_merge( $this->codenames_old, $this->codenames );
foreach ( $debinfo as $di ) {
echo 'Mismatching distribution "' . $di['info']['distr'] . '" in ' . $di['deb']['file'] . ':' . $di['deb']['line'] . PHP_EOL;
$better = array();
$current = array_search( $di['info']['distr'], $all_codenames );
foreach ( $di['server']['dists'] as $dist_avail ) {
$where = array_search( $dist_avail, $all_codenames );
if ( $where === false || $where > $current ) {
$better[] = $dist_avail;
}
}
//echo 'Available: ' . implode( ', ', $di['server']['dists'] ) . PHP_EOL;
if ( count( $better ) > 0 ) echo 'Possibly better matches: ' . implode( ', ', $better ) . PHP_EOL;
}
}
}
$ac = new APTChecker();
$ac->addWhitelist( array( 'stable', 'unstable', 'beta' ) );
echo 'System: ' . $ac->getCodename() . PHP_EOL;
echo 'Running as user id ' . posix_getuid() . PHP_EOL;
echo 'Parsing lists ... ';
$debs = $ac->parseLists();
echo count($debs) . ' entries found.' . PHP_EOL;
if ( $ac->isRoot() ) {
echo 'Fetching key ids of keyring ... ';
exec('apt-key list', $output, $retval);
$output = implode(PHP_EOL, $output);
preg_match_all('/pub.*\/([^\s]+).*\r?\n/m', $output, $matches);
$allkeys = $matches[1];
echo count($allkeys) . ' keys found.' . PHP_EOL;
} else {
echo 'Not started as root. Will not check GPG keys.' . PHP_EOL;
}
$debinfo = $ac->analyzeDebs( $debs, 'progressInd' );
echo PHP_EOL;
$ac->outputResults( $debinfo );
if ( $ac->isRoot() ) {
foreach ( $keycache as $url=>$dists ) {
foreach ( $dists as $dist=>$key ) {
if ( !empty( $key['id'] ) && !in_array( $key['id'], $allkeys ) ) {
echo 'Importing key ' . $key['id'] . ' ... ';
passthru( 'apt-key adv --batch --recv-keys --keyserver keyserver.ubuntu.com ' . $key['id'] );
}
}
}
}
echo 'All done.';
if ( !$ac->isRoot() ) echo ' (Run as root to import missing PPA keys.)';
echo PHP_EOL;
exit;
function progressInd() {
echo '.';
}
function checkCurrentDist($url) {
$fullurl = 'http://linux.getdropbox.com/ubuntu/dists/jaunty/main/binary-i386/Packages.gz';
}

116
apt-urlcheck.py Executable file
View File

@ -0,0 +1,116 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import ansi
import os.path
import re
import requests
from aptsources.distro import get_distro
from aptsources.sourceslist import SourcesList
from typing import TypeVar
codenames_okay = ["devel", "stable", "unstable", "beta", "preview", "testing", "syncthing"]
codenames_old = ["gutsy", "hardy", "intrepid", "jaunty", "karmic", "lucid", "maverick", "natty", "oneiric", "precise", "quantal", "raring", "saucy", "trusty", "utopic", "vivid", "wily"]
codenames = ["jessie", "xenial", "yakkety", "zesty", "stretch", "artful", "bionic", "squeeze"]
codename = get_distro().codename
print("This is {}.".format(ansi.YELLOW + codename + ansi.RESET))
print("Loading sources...", end="", flush=True)
valid_sources = 0
outdated_sources = 0
check_sources = []
for source in SourcesList():
if source.disabled or source.line=="\n":
continue
valid_sources += 1
#print(".", end="", flush=True)
if codename in source.dist or source.dist in codenames_okay:
continue
outdated_sources += 1
check_sources.append(source)
#print("{}: {}".format(source.file, source.line.strip()))
check_sources = sorted(check_sources, key=lambda x: x.file)
print(" OK")
print("Found {} sources with {} possibly outdated.".format(valid_sources, outdated_sources))
BorL = TypeVar("BorL", bool, list)
fetch_cache = {}
def try_fetch_dirlisting(url: str) -> BorL:
global codenames_okay, codenames_old, codenames, fetch_cache
if url in fetch_cache:
return fetch_cache[url]
all_known_codenames = codenames_old + codenames
result = requests.get(url)
if result.status_code != 200:
fetch_cache[url] = False
return False
matches = re.findall(r"<a .*?href=['\"]?([^'\"]+)['\"]?.*?>", result.text)
valid_matches = []
for match in matches:
if match[0] != "?" and match[0] != "/" and match[-1:] == "/" and match != "../" and match[0:4] != "http":
valid_matches.append(match[0:-1])
elif match in all_known_codenames or match[0:-1] in all_known_codenames:
valid_matches.append(match)
fetch_cache[url] = valid_matches
return fetch_cache[url]
def mutate_codename(current_codename: str, new_codename: str) -> str:
global codenames_okay, codenames_old, codenames
all_codenames = codenames_old + codenames
for cn in all_codenames:
if cn in current_codename:
return current_codename.replace(cn, new_codename)
return new_codename
probe_cache = {}
def try_url_probing(url: str, current_codename: str) -> list:
global codenames_okay, codenames_old, codenames, probe_cache
cache_key = url + "|" + current_codename
if cache_key in probe_cache:
return probe_cache[cache_key]
test_set = codenames + codenames_okay
valid_matches = []
for codename in test_set:
for filename in ["InRelease", "Release", "Release.gpg"]:
mcodename = mutate_codename(current_codename, codename)
try_url = "{}/{}/{}".format(url, mcodename, filename)
print(".", end="", flush=True)
result = requests.get(try_url)
if result.status_code == 200:
valid_matches.append(mcodename)
break
probe_cache[cache_key] = valid_matches
return probe_cache[cache_key]
def filter_better_matches(found_codenames: list, current_codename: str) -> list:
global codenames, codenames_okay, codenames_old
better_codenames = []
for cn in found_codenames:
if cn > current_codename:
better_codenames.append(cn)
elif cn in codenames_okay:
better_codenames.append(cn)
return better_codenames
for src in check_sources:
print("{}: Outdated codename: {}".format(ansi.CYAN + os.path.basename(src.file) + ansi.RESET, ansi.RED + src.dist + ansi.RESET))
test_url = src.uri + "/dists"
more_options = try_fetch_dirlisting(test_url)
if not more_options:
print("Listing failed. Probing", end="", flush=True)
more_options = try_url_probing(test_url, src.dist)
print(" OK")
print(ansi.UP_DEL, end="")
better_options = filter_better_matches(more_options, src.dist)
if better_options:
print("Possibly better options: {}".format(ansi.GREEN + (ansi.RESET + ", " + ansi.GREEN).join(better_options) + ansi.RESET))
else:
print(ansi.SILVER + "No better match(es) found at the moment." + ansi.RESET)

3
requirements.txt Normal file
View File

@ -0,0 +1,3 @@
# Install via aptitude, pip doesn't work!
python-apt
python-distutils-extra