#!/usr/bin/python3
# -*- coding: utf-8 -*-
# pylint: disable=line-too-long
# kate: space-indent on; indent-width 4; replace-tabs on; indent-mode python; remove-trailing-space modified;
# vim: expandtab ts=4
# pylint: enable=line-too-long
############################################################################
# Copyright © 2015-2019 José Manuel Santamaría Lema <panfaust@gmail.com> #
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
############################################################################
"""
This module provides utilities to deal with the upstream FTP's where
source code tarballs are published
"""
import json
import os
import sys
import subprocess
import re
from ftplib import FTP
from io import StringIO
from debian.debian_support import Version
from libka.ka_configuration import get_ka_config_parser
from libka.ka_data_utils import read_json_data_file
from libka.qtwebkit_releases_html_parser import QtWebKitReleasesHTMLParser
from libka.ka_print import ka_print_plain
from libka.ka_print import ka_print_warning
from libka.ka_print import ka_print_error
[docs]
def getFtpVersionMap(releaseType): #pylint: disable=invalid-name
"""DEPRECATED: use `get_ftp_version_map` instead"""
return get_ftp_version_map(releaseType)
[docs]
def get_ftp_version_map(release_type):
#pylint: disable=too-many-branches,too-many-statements,too-many-locals
"""
Returns a map where the key is the upstream tarball name and the value
is the version.
The `release_type` parameter is the release type; "qt", "frameworks",
"plasma" or "applications".
"""
#Read configuration
ka_config = get_ka_config_parser()
kde_sftp_host = ka_config['global']['kde-sftp-host']
#Populate and return the map
package_version_map = {}
#Find out the version
rt_versions_map = read_json_data_file("versions.json")
version = Version(rt_versions_map[release_type])
#If the result is cached return it
cache_dir = "~/.kubuntu-automation/cache/kde_ftp/"
cache_dir = os.path.expanduser(cache_dir)
cache_file_path = os.path.join(cache_dir, "%s-%s.json" % (release_type, version))
try:
cache_file = open(cache_file_path, 'r')
package_version_map = json.load(cache_file)
return package_version_map
except FileNotFoundError:
pass
#Find out which subdirectories we have to inspect in the ftp
ftp_subdirs = []
if release_type == "frameworks":
ftp_subdirs = ["", "portingAids"]
elif release_type == "plasma":
ftp_subdirs = [""]
elif release_type == "applications":
ftp_subdirs = ["src"]
elif release_type == "qt":
ftp_subdirs = [""]
#Find out the stability
version_parts = str(version).split(".")
last_digit = int(version_parts[-1])
if release_type != "frameworks" and last_digit >= 80:
stability = "unstable"
else:
stability = "stable"
#Inspect the ftp
for subdir in ftp_subdirs:
if release_type != "qt":
kde_sftp_host = ka_config['global']['kde-sftp-host']
if release_type == "applications" and version >= Version('19.12'):
ftp_command = ["sftp", "-b", "-", "%s:%s/%s/%s/%s"
% (kde_sftp_host, stability, "release-service", version, subdir)]
else:
ftp_command = ["sftp", "-b", "-", "%s:%s/%s/%s/%s"
% (kde_sftp_host, stability, release_type, version, subdir)]
devnull = open(os.devnull, 'w')
process = subprocess.Popen(ftp_command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=devnull)
output, _ = process.communicate(bytes("ls *xz", 'utf-8'))
if process.returncode != 0:
ka_print_warning("WARNING: Accessing FTP listing via plain FTP.")
ka_print_warning("Maybe you want to configure your ssh config to access %s?"
% kde_sftp_host)
kde_ftp = "ftp.pbone.net"
kde_ftp_dir = "pub/kde"
ftp = FTP(kde_ftp)
try:
ftp.login()
except Exception as exception: #pylint: disable=broad-except
ka_print_error("Can't login to %s" % kde_ftp)
ka_print_error(str(exception))
sys.exit(1)
try:
ftp.cwd(kde_ftp_dir)
except Exception as exception: #pylint: disable=broad-except
ka_print_error("Can't access %s directory on %s" % (kde_ftp_dir, kde_ftp))
ka_print_error(str(exception))
sys.exit(1)
if release_type == "applications" and version >= Version('19.12'):
kde_ftp_subdir = "%s/%s/%s/%s" % (stability, "release-service", version, subdir)
else:
kde_ftp_subdir = "%s/%s/%s/%s" % (stability, release_type, version, subdir)
try:
ftp.cwd(kde_ftp_subdir)
except Exception as exception: #pylint: disable=broad-except
ka_print_error("Can't access %s directory on %s" % (kde_ftp_subdir, kde_ftp))
ka_print_error(str(exception))
sys.exit(1)
output = StringIO()
callback = lambda s: output.write(s+'\n') #pylint: disable=no-member,cell-var-from-loop
ftp.retrlines("NLST *.xz", callback)
output = bytes(output.getvalue(), 'utf-8')
else:
#Get the FTP listing of all 'standard' Qt packages
ftp = FTP("ftp.mirrorservice.org")
ftp.login()
version_short = version_parts[0] + "." + version_parts[1]
ftp_dir = ("sites/download.qt-project.org/official_releases/qt/%s/%s/submodules/"
% (version_short, version))
ftp.cwd(ftp_dir)
output = StringIO()
callback = lambda s: output.write(s+'\n') #pylint: disable=no-member,cell-var-from-loop
ftp.retrlines("NLST *.xz", callback)
output = bytes(output.getvalue(), 'utf-8')
#Handle qtwebkit as a special case
qtwebkit_html_parser = QtWebKitReleasesHTMLParser()
qtwebkit_html_parser.parse_releases_url()
package_version_map['qtwebkit'] = str(qtwebkit_html_parser.get_latest_version())
#Fill the map inspecting the "ls" results from the ftp
for line in output.splitlines():
line = line.decode('utf-8')
match = re.search(r'([a-zA-Z0-9\-]+)-' + r'([\d.]*)' + r'\.tar\.', line)
if match:
package = match.group(1)
package_version = match.group(2)
package_version_map[package] = package_version
#Store the result in cache
try:
os.makedirs(cache_dir)
except: #pylint: disable=bare-except
pass
cache_file = open(cache_file_path, 'w')
json.dump(package_version_map, cache_file)
cache_file.close()
#Return the resulting map
return package_version_map
[docs]
def download_tarballs_from_kde_sftp(release_type):
#pylint: disable=too-many-branches,too-many-statements,too-many-locals
"""
Downloads all the tarballs from depot for the specified release type.
Returns `True` if the tarballs were sucessfully downloaded.
"""
#Read configuration and data files
ka_config = get_ka_config_parser()
config_locations = ka_config['tarball-locations']
rt_versions = read_json_data_file("versions.json")
#Read KDE's SFTP data
kde_sftp_host = ka_config['global']['kde-sftp-host']
kde_sftp_root_dir = ka_config['global']['kde-sftp-root-dir']
#Prepare the destination directory
if release_type != "kde-l10n":
version = Version(rt_versions[release_type])
else:
version = Version(rt_versions["applications"])
dest_dir = os.path.join(config_locations[release_type], str(version))
dest_dir = os.path.expanduser(dest_dir)
os.makedirs(dest_dir, exist_ok=True)
#Find out the stability
version_parts = str(version).split(".")
last_digit = int(version_parts[-1])
if release_type != "frameworks" and last_digit >= 80:
stability = "unstable"
else:
stability = "stable"
#Find out which subdirectories we have to inspect in the ftp
ftp_subdirs = []
if release_type == "frameworks":
ftp_subdirs = ["", "portingAids/"]
elif release_type == "plasma":
ftp_subdirs = [""]
elif release_type == "applications":
ftp_subdirs = ["src/"]
elif release_type == "kde-l10n":
release_type = "applications"
ftp_subdirs = ["src/kde-l10n/"]
#Download tarballs for all subdirectories
url_template = "%s:%s"
tarball_downloaded = False
for subdir in ftp_subdirs:
if release_type == "applications" and version >= Version('19.12'):
kde_sftp_path = os.path.join(kde_sftp_root_dir, stability,
"release-service", str(version), subdir)
else:
kde_sftp_path = os.path.join(kde_sftp_root_dir, stability,
release_type, str(version), subdir)
url = url_template % (kde_sftp_host, kde_sftp_path)
try:
download_command = ["rsync", "-av", "--progress",
"--include=%s" % kde_sftp_path,
"--include=*.tar.xz",
"--exclude=*",
url, dest_dir]
ka_print_plain(" ".join(download_command))
sys.stdout.flush()
subprocess.check_call(download_command)
sys.stdout.flush()
tarball_downloaded = True
#Also download upstream sig files if configured
if ka_config['tarball-locations'].getboolean('upstream-tarball-sigs'):
download_command = ["rsync", "-av", "--progress",
"--include=%s" % kde_sftp_path,
"--include=*.tar.xz.sig",
"--exclude=*",
url, dest_dir]
ka_print_plain(" ".join(download_command))
sys.stdout.flush()
subprocess.check_call(download_command)
sys.stdout.flush()
except: #pylint: disable=bare-except
pass
return tarball_downloaded