#!/usr/bin/python3
# -*- coding: utf-8 -*-
# pylint: disable=line-too-long
# kate: space-indent on; indent-width 4; replace-tabs on; indent-mode python; remove-trailing-space modified;
# vim: expandtab ts=4
# pylint: enable=line-too-long
# pylint: disable=too-many-lines
############################################################################
# Copyright © 2015-2024 José Manuel Santamaría Lema <panfaust@gmail.com> #
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
############################################################################
"""This module just provides the KASrcPkg class"""
import sys
import os
import stat
import tempfile
import re
import subprocess
import shutil
from launchpadlib.launchpad import Launchpad
from debian.debian_support import Version
from libka.ka_configuration import get_ka_config_parser
from libka.pkgedit.ka_control_file import KAControlFile
from libka.pkgedit.ka_changelog import KAChangelog
from libka.pkgedit.ka_watch_file import KAWatchFile
from libka.kubuntu_ppa import KubuntuPPA
from libka.tritemiolib import tritemio_ls
from libka.kde_ftp import get_ftp_version_map
from libka.ka_data_utils import read_json_data_file
from libka.ka_data_utils import read_packages_file
from libka.ka_data_utils import get_data_files_path
from libka.utils.verify_signature import verify_signature
from libka.utils.get_orig_tarball_extension import get_orig_tarball_extension
from libka.ka_print import ka_print_good_stuff
from libka.ka_print import ka_print_plain
from libka.ka_print import ka_print_error
[docs]
def get_release_type(upstream_tarball_name, use_overrides=True):
"""
This function returns the release type (qt, frameworks, plasma, applications, other)
for the package given by `upstream_tarball_name`
"""
#Return the release type right away if this package is listed in the overrides file
if use_overrides:
overrides_file_name = "release-type-overrides.json"
json_map = read_json_data_file(overrides_file_name)
if upstream_tarball_name in json_map:
return json_map[upstream_tarball_name]
#Handle some special cases for Qt
if upstream_tarball_name.endswith("-opensource-src"):
upstream_tarball_name = upstream_tarball_name.split("-opensource-src")[0]
upstream_tarball_name += "-everywhere-src"
#Given the tarball name, let's see if we have it in any ftp listing
release_types_to_check = ["frameworks", "plasma", "applications", "qt"]
release_types = []
number_of_release_types = 0
for release_type in release_types_to_check:
if upstream_tarball_name in get_ftp_version_map(release_type):
release_types.append(release_type)
number_of_release_types += 1
#Return result
if number_of_release_types == 0:
#If we didn't find the release type so far, let's supose it belongs to 'other'
result = "other"
elif number_of_release_types == 1:
result = release_types[0]
else:
overrides_file_path = os.path.join(get_data_files_path(), overrides_file_name)
ka_print_error("ERROR: This package belongs to more than one release type according to "
"the FTP listings")
ka_print_plain("(release types detected = %s)\n"
"Please edit the file %s to pick the correct release type for this package"
% (release_types, overrides_file_path))
sys.exit(1)
return result
# pylint: disable=too-many-public-methods,too-many-instance-attributes
[docs]
class KASrcPkg():
"""
This class is meant to represent a Debian source package, extract
information about the package and change it.
"""
def __init__(self):
#Check if we have at least a changelog and control file
if not os.path.isfile('debian/changelog'):
raise Exception("Couldn't find debian/changelog")
if not os.path.isfile('debian/control'):
raise Exception("Couldn't find debian/control")
#Init internal variables
self._control_file_loaded = False
self._control_file = None
self._tests_control_file_loaded = False
self._tests_control_file = None
self._changelog_file_loaded = False
self._changelog = None
self._watch_file_loaded = False
self._watch_file = None
[docs]
def load_control_file(self):
"""This function opens the debian/control file and parses it so its data can be
used in other parts of the KaSrcPkg class"""
self._control_file = KAControlFile()
try:
self._control_file.parse_control()
except Exception as exception:
raise Exception("Couldn't open debian/control\n" + str(exception))
self._control_file_loaded = True
[docs]
def load_tests_control_file(self):
"""This function opens the debian/tests/control file and parses it so its data can be
used in other parts of the KaSrcPkg class"""
self._tests_control_file = KAControlFile('debian/tests/control')
try:
self._tests_control_file.parse_control()
except FileNotFoundError:
self._tests_control_file = None
except Exception as exception:
raise Exception("Couldn't open debian/tests/control\n" + str(exception))
self._tests_control_file_loaded = True
[docs]
def load_changelog(self):
"""This function opens the debian/changelog file and parses it so its data can
be used in other parts of the KASrcPkg class"""
self._changelog = KAChangelog()
self._changelog.parse_changelog()
self._changelog_file_loaded = True
[docs]
def load_watch_file(self):
"""
This function opens the `debian/watch` file and parses it so its data can be used
in other parts of the `KASrcPkg` class.
"""
if not self.is_native_package():
self._watch_file = KAWatchFile()
self._watch_file.parse_watch()
self._watch_file_loaded = True
[docs]
def reload(self):
"""
This function loads all files again.
This is useful if the files changed or if you are using this class against a git
repository and you checkout another branch and you need to refresh the data.
"""
if self._control_file_loaded:
self.load_tests_control_file()
if self._tests_control_file_loaded:
self.load_control_file()
if self._changelog_file_loaded:
self.load_changelog()
if self._watch_file_loaded:
self.load_watch_file()
[docs]
def sanitize_control_file(self, file_name='debian/control'): #pylint: disable=no-self-use
"""
Sanitizes a debian/control file so it won't trigger bugs from python-debian
or wrap-and-sort.
"""
control_file = KAControlFile(file_name)
control_file.sanitize()
[docs]
def get_version(self):
"""
Returns the package version from changelog.
"""
if not self._changelog_file_loaded:
self.load_changelog()
return self._changelog.get_version()
[docs]
def get_last_released_version(self):
"""
This function returns the last released version i.e. the version from the
most recent changelog block ignoring 'UNRELEASED' changelog blocks.
"""
if not self._changelog_file_loaded:
self.load_changelog()
return self._changelog.get_last_released_version()
[docs]
def get_last_ubuntu_released_version(self):
"""
This function returns the last released version i.e. the version from the
most recent changelog block ignoring 'UNRELEASED' and Debian changelog blocks.
"""
if not self._changelog_file_loaded:
self.load_changelog()
return self._changelog.get_last_ubuntu_released_version()
[docs]
def source_package_name(self):
"""
Returns the name of the current source package.
"""
if not self._changelog_file_loaded:
self.load_changelog()
return self._changelog.get_package()
[docs]
def get_bin_packages(self):
"""
Returns the list of binary package names produced by this source package.
"""
if not self._control_file_loaded:
self.load_control_file()
return self._control_file.get_bin_packages()
[docs]
def upstream_tarball_name(self):
"""Returns the upstream tarball name reading it from debian/watch"""
if not self._watch_file_loaded:
self.load_watch_file()
return self._watch_file.tarball_name(self.source_package_name())
[docs]
def upstream_tarball_version(self):
"""
Returns the upstream tarball version
"""
#Find out release type
release_type = self.get_release_type()
#Find out upstream version
upstream_version = self.get_version().upstream_version
#Handle packages with updated tarballs
#e.g. 5.0.0a version instead of 5.0.0
#or 5.11.1+dfsg2 instead of 5.11.1+dfsg
tarball_repacked = self.upstream_tarball_repacked()
last_char = upstream_version[-1]
last_char_must_be_removed = False
if release_type in ["qt", "frameworks", "plasma", "applications"]:
last_char_must_be_removed = (last_char.isalpha() and not tarball_repacked)
if last_char_must_be_removed:
tarball_version = upstream_version.rstrip(last_char)
else:
tarball_version = upstream_version
#Return result
return tarball_version
[docs]
def upstream_tarball_repacked(self):
"""Returns `True` if the upstream tarball is being repacked"""
if not self._watch_file_loaded:
self.load_watch_file()
return self._watch_file.tarball_repacked()
[docs]
def upstream_tarball_repack_suffix(self):
"""Returns the upstream tarball repack suffix (e.g. `+dfsg`)"""
if not self._watch_file_loaded:
self.load_watch_file()
return self._watch_file.tarball_repack_suffix()
[docs]
def upstream_git_name(self):
"""Returns the upstream (KDE) git name."""
#Find out the tarball name
upstream_tarball_name = self.upstream_tarball_name()
#Find out the git name managing exceptional cases manually listed in ka-metadata
kde_git_map = read_json_data_file('name-conversions/kde-tarball-to-kde-git.json')
if upstream_tarball_name in kde_git_map:
upstream_git_name = kde_git_map[upstream_tarball_name]
else:
upstream_git_name = upstream_tarball_name
#Return result
return upstream_git_name
[docs]
def get_release_type(self, use_overrides=True):
"""
Find out the release type for a given package.
Returns the release type string: "frameworks", "plasma", "applications", "qt" or "other".
"""
#Try to find out the tarball name
try:
upstream_tarball_name = self.upstream_tarball_name()
except: # pylint: disable=bare-except
return "other"
#Use the tarball name to find out the release type
return get_release_type(upstream_tarball_name, use_overrides)
[docs]
def is_native_package(self):
"""Returns `True` if the package is native"""
return self.get_source_format() == "3.0 (native)"
[docs]
def get_patches_list(self):
"""Returns de patches series list"""
#First of all check that source format is compatible with quilt
source_format = self.get_source_format()
if not source_format.endswith("(quilt)"):
raise Exception("Error gettting the patches list, "
"the source format is not 'x.y (quilt)'")
#Read the contents of the 'series' file
patches_list = []
try:
with open('debian/patches/series', 'r') as series_file:
lines = series_file.readlines()
for line in lines:
line = line.strip()
if line.startswith('#'):
continue
else:
patches_list.append(line)
except FileNotFoundError:
patches_list = []
#Return result
return patches_list
[docs]
def get_standards_version(self):
"""Returns the current Standards-Version as a Version object"""
if not self._control_file_loaded:
self.load_control_file()
return self._control_file.get_standards_version()
[docs]
def set_standards_version(self, new_value):
"""Sets the current Standards-Version"""
if not self._control_file_loaded:
self.load_control_file()
self._control_file.set_standards_version(new_value)
[docs]
def ka_wrap_and_sort(self):
#TODO
pass
# pylint: disable=no-self-use
[docs]
def fix_rules_perms(self):
"""Calling this function will set debian/rules permissions to 0755.
Returns False if the permissions are already 0755, otherwise returns True."""
if oct(os.stat('debian/rules')[stat.ST_MODE])[-3:] == '755':
result = False
else:
os.chmod('debian/rules', 0o755)
result = True
return result
# pylint: enable=no-self-use
[docs]
def add_gitattributes_file(self):
#TODO
pass
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
[docs]
def download_tarball_from_kde_sftp(self):
"""
Downloads the orig source tarball from KDE's sftp.
"""
#Abort if the package is native
if self.is_native_package():
raise Exception("Trying to use download_tarball_from_kde_sftp() "
"against a native package")
#Read configuration files
ka_config = get_ka_config_parser()
#Read KDE's SFTP data
kde_sftp_host = ka_config['global']['kde-sftp-host']
kde_sftp_root_dir = ka_config['global']['kde-sftp-root-dir']
#Find out release type
release_type = self.get_release_type()
#Find out upstream tarball version
tarball_version = self.upstream_tarball_version()
#Remove repacking suffixes from tarball version
if self.upstream_tarball_repacked():
repack_suffix = self.upstream_tarball_repack_suffix()
repack_suffix_escaped = re.escape(repack_suffix)
tarball_version = re.sub(repack_suffix_escaped + '$', '', tarball_version)
#Find out dir version and last digit
version_parts = tarball_version.split(".")
dir_version = tarball_version
try:
if release_type == "frameworks":
expected_format = "x.y"
dir_version = version_parts[0] + "." + version_parts[1]
last_digit = int(version_parts[-1])
else:
expected_format = "x.y.z"
dir_version = "%s.%s.%s" % (version_parts[0], version_parts[1], version_parts[2])
last_digit = int(version_parts[2])
except Exception as exception: #pylint: disable=bare-except
ka_print_error("Failed to parse upstream tarball version '%s': "
"it doesn't match the expected format '%s'"
% (tarball_version, expected_format))
sys.exit(1)
#Prepare the destination directory
dest_dir = os.path.join(ka_config['tarball-locations'][release_type], dir_version)
dest_dir = os.path.expanduser(dest_dir)
os.makedirs(dest_dir, exist_ok=True)
#Find out the stability
if last_digit >= 80:
stability = "unstable"
else:
stability = "stable"
#Find out which subdirectories we have to inspect in the ftp
ftp_subdirs = []
if release_type == "frameworks":
ftp_subdirs = ["", "portingAids/"]
elif release_type == "plasma":
ftp_subdirs = [""]
elif release_type == "applications":
ftp_subdirs = ["src/"]
#Create url template to access it later
url_template = "%s:%s/%s/%s/%s/%s%s-%s.tar.xz"
#Find out tarball name
tarball_name = self.upstream_tarball_name()
#Inspect the sftp and download the tarballs
tarball_downloaded = False
for subdir in ftp_subdirs:
if release_type == "applications" and Version(dir_version) >= Version('19.12'):
url = url_template % (kde_sftp_host, kde_sftp_root_dir,
stability, "release-service",
dir_version, subdir,
tarball_name, tarball_version)
else:
url = url_template % (kde_sftp_host, kde_sftp_root_dir,
stability, release_type,
dir_version, subdir,
tarball_name, tarball_version)
try:
#Download the upstream tarball
download_command = ["rsync", "-av", "--progress", url, dest_dir]
ka_print_plain(" ".join(download_command))
sys.stdout.flush()
subprocess.check_call(download_command)
sys.stdout.flush()
#Also download upstream sig file if configured
if ka_config['tarball-locations'].getboolean('upstream-tarball-sigs'):
download_command = ["rsync", "-av", "--progress", url + '.sig', dest_dir]
ka_print_plain(" ".join(download_command))
sys.stdout.flush()
subprocess.check_call(download_command)
sys.stdout.flush()
#Verify if the tarball was actually downloaded, rsync sometimes
#returns 0 (sucess) even if the remote file doesn't exist on the
#KDE SFTP, therefore the above subprocess
tarball_full_name = tarball_name + '-' + tarball_version + ".tar.xz"
tarball_download_path = os.path.join(dest_dir, tarball_full_name)
if os.path.isfile(tarball_download_path):
tarball_downloaded = True
except: #pylint: disable=bare-except
pass
#Repack the tarball if needed
if self.upstream_tarball_repacked():
#ka_print_info("Repacking tarball...")
print("Repacking tarball...")
sys.stdout.flush()
sftp_tarball_file_name = "%s-%s.tar.xz" % (tarball_name, tarball_version)
sftp_tarball_full_path = os.path.join(dest_dir, sftp_tarball_file_name)
repacked_tarball_file_name = "%s_%s.orig.tar.xz" % (tarball_name, self.upstream_tarball_version())
repacked_tarball_full_path = os.path.join(dest_dir, repacked_tarball_file_name)
print(sftp_tarball_full_path) #FIXME: debug
print(repacked_tarball_full_path) #FIXME: debug
#TODO: open copyright file and parse
raise Exception("Not implemented yet")
#Return result
return tarball_downloaded
# pylint: enable=too-many-locals,too-many-statements,too-many-branches
[docs]
def download_tarball_with_uscan(self, repack=False):
"""
Downloads the orig source tarball with uscan.
"""
#Abort if the package is native
if self.is_native_package():
raise Exception("Trying to use download_tarball_with_uscan() against a native package")
#Read configuration files
ka_config = get_ka_config_parser()
#Find out release type
release_type = self.get_release_type()
#Create destination directory
destdir = ka_config['tarball-locations'][release_type]
if release_type != "other":
upstream_version = self.get_version().upstream_version
dir_version = upstream_version
#Replace ~ for alpha and beta versions
dir_version = re.sub(r'~', r'-', dir_version)
#Remove repacking suffixes from tarball version
if self.upstream_tarball_repacked():
repack_suffix = self.upstream_tarball_repack_suffix()
dir_version = re.sub(repack_suffix + '$', '', dir_version)
#Parse version parts
version_parts = upstream_version.split(".")
#Find out the directory name
if release_type == "frameworks":
dir_version = version_parts[0] + "." + version_parts[1]
elif release_type == "plasma":
dir_version = "%s.%s.%s" % (version_parts[0], version_parts[1], version_parts[2])
#Compose the full path of the destination directory
destdir = os.path.join(destdir, dir_version)
destdir = os.path.expanduser(destdir)
os.makedirs(destdir, exist_ok=True)
#Download tarball to the destination directory.
program = ["uscan"]
if repack:
program += ["--repack", "--compression=xz"]
else:
program += ["--no-symlink"]
#FIXME: this part is commented out because this causes the tarball to not be downloaded
# if you are downloading with uscan and upstream tarball signatures are disabled
# this is probably a bug in uscan
#if not ka_config['tarball-locations'].getboolean('upstream-tarball-sigs'):
# program += ["--skip-signature"]
program += ["--download-current-version", "--destdir="+destdir]
ka_print_plain(" ".join(program))
try:
sys.stdout.flush()
subprocess.check_call(program)
sys.stdout.flush()
tarball_downloaded = True
except subprocess.CalledProcessError:
ka_print_error("uscan failed")
tarball_downloaded = False
return tarball_downloaded
[docs]
def upstream_tarball_paths(self):
"""
Returns the path where the upstream tarball should be available to be symlinked for package
building.
For example:
.. code-block:: python
[ "/home/user/kde-ftp/frameworks/5.81/extra-cmake-modules-5.81.0.tar.xz",
"/home/user/kde-ftp/frameworks/5.81/extra-cmake-modules-5.81.0.tar.gz",
...
]
This is going to be the target of the symlink in the 'build-area' directory.
"""
#Abort if the package is native
if self.is_native_package():
raise Exception("Trying to use get_orig_tarball() against a native package")
#Read configuration files
ka_config = get_ka_config_parser()
#Find out release type
release_type = self.get_release_type()
#Find out upstream tarball version
tarball_version = self.upstream_tarball_version()
#Find out dir version, to find out later the tarball dir
if release_type == "other":
dir_version = ""
elif release_type == "frameworks":
version_parts = tarball_version.split(".")
dir_version = version_parts[0] + "." + version_parts[1]
else:
version_parts = tarball_version.split(".")
dir_version = "%s.%s.%s" % (version_parts[0], version_parts[1], version_parts[2])
#Remove repacking suffixes from dir version
if self.upstream_tarball_repacked():
repack_suffix = self.upstream_tarball_repack_suffix()
dir_version = re.sub(repack_suffix + '$', '', dir_version)
#Find out real orig tarball dir
tarball_dir = os.path.expanduser(ka_config['tarball-locations'][release_type])
tarball_dir = os.path.join(tarball_dir, dir_version)
#Get the list of possible tarball names
tarball_names_list = self._watch_file.tarball_patterns(self.source_package_name(),
tarball_version)
#For each tarball name, build the full real path
tarball_paths = []
for tarball_name in tarball_names_list:
tarball_path = os.path.join(tarball_dir, tarball_name)
#Replace ~ for alpha and beta versions
tarball_path = re.sub(r'~', r'-', tarball_path)
tarball_paths.append(tarball_path)
#Return result
return tarball_paths
[docs]
def get_orig_tarball_path(self, tarball_extension):
"""
Returns the path where the orig tarball should be available for package building.
For example: `../build-area/foo_5.1.orig.tar.xz`
"""
#Abort if the package is native
if self.is_native_package():
raise Exception("Trying to use get_orig_tarball_path() against a native package")
#Read configuration files
ka_config = get_ka_config_parser()
#Find out package info - src package name, upstream version
src_package_name = self.source_package_name()
upstream_version = self.get_version().upstream_version
#Find out the build area dir and create it if it doesn't exist
build_area_dir = ka_config['areas']['build-area']
os.makedirs(build_area_dir, exist_ok=True)
#Find out the file name
link_name = src_package_name + '_' + upstream_version + '.orig.' + tarball_extension
#Create the complete path
link_path = os.path.join(build_area_dir, link_name)
#Return result
return link_path
[docs]
def get_orig_tarball(self): #pylint: disable=too-many-locals,too-many-branches
"""
Downloads and creates as symlink for the orig tarball.
"""
#Abort if the package is native
if self.is_native_package():
raise Exception("Trying to use get_orig_tarball() against a native package")
#Read configuration files
ka_config = get_ka_config_parser()
#Find out release type
release_type = self.get_release_type()
#Find out upstream version
upstream_version = self.get_version().upstream_version
#Find out if we should use KDE's sftp or not
download_tarball_from_kde_sftp = False
if ka_config['tarball-locations'].getboolean('use-kde-sftp'):
if release_type in ["frameworks", "plasma", "applications"]:
download_tarball_from_kde_sftp = True
#Try to download the tarball from KDE's sftp
if download_tarball_from_kde_sftp:
ka_print_good_stuff("Syncing orig tarball from KDE SFTP...")
sys.stdout.flush()
downloaded_with_sftp = self.download_tarball_from_kde_sftp()
else:
downloaded_with_sftp = False
#Download the tarball with uscan (if it wasn't already)
if not downloaded_with_sftp:
ka_print_good_stuff("Downloading orig tarball with uscan...")
sys.stdout.flush()
#Repack the orig tarball if needed
repack = self.upstream_tarball_repacked()
#Download the tarball
self.download_tarball_with_uscan(repack)
#Find out symlink possible targets
symlink_target_paths = self.upstream_tarball_paths()
#Iterate over the symlink possible targets and create the symlink as soon
#as we find one that points to a valid file
symlink_created = False
symlink_path = None
for symlink_target_path in symlink_target_paths: #pylint: disable=too-many-nested-blocks
if os.path.exists(symlink_target_path):
symlink_created = True
#Find out tarball extension
tarball_extension = get_orig_tarball_extension(symlink_target_path)
#Find out symlink path
symlink_path = self.get_orig_tarball_path(tarball_extension)
#Create the symlink, overwriting any old symlink
os.makedirs(ka_config['areas']['build-area'], exist_ok=True)
if os.path.exists(symlink_path):
os.remove(symlink_path)
os.symlink(symlink_target_path, symlink_path)
#Create a symlink to the upstream signature if configured
if ka_config['tarball-locations'].getboolean('upstream-tarball-sigs'):
watch_file_opts = self._watch_file.opts()
for watch_file_opts_dict in watch_file_opts:
symlink_target_path_sig = None
if 'pgpsigurlmangle' in watch_file_opts_dict:
regex_string = watch_file_opts_dict['pgpsigurlmangle'] #pylint: disable=invalid-sequence-index
try:
#This complicated code below executes the pggsirurlmangle regex in
#the watch file, it's tipically something like s/$/.sig/ so the
#result in practice would be just appending '.sig' to the tarball
#names
regex_string_parts = re.match(r's/(.*[^/])/([^/].*)/', regex_string)
regex_string_pattern = regex_string_parts.group(1)
regex_string_repl = regex_string_parts.group(2)
symlink_target_path_sig = re.sub(regex_string_pattern,
regex_string_repl,
symlink_target_path)
symlink_path_sig = re.sub(regex_string_pattern,
regex_string_repl,
symlink_path)
except Exception as exception: # pylint: disable=broad-except
ka_print_error("Error parsing pggsirurlmangle from debian/watch")
ka_print_error(str(exception))
break
if 'pgpmode' in watch_file_opts_dict:
if watch_file_opts_dict['pgpmode'] == 'auto':
symlink_target_path_sig = symlink_target_path + '.sig'
symlink_path_sig = symlink_path + '.sig'
if symlink_target_path_sig is not None:
if os.path.exists(symlink_target_path_sig):
if os.path.exists(symlink_path_sig):
os.remove(symlink_path_sig)
os.symlink(symlink_target_path_sig, symlink_path_sig)
#Verify signature if we downloaded from KDE's SFTP,
#when downloading with uscan this gets automatically done
if downloaded_with_sftp:
verify_signature(symlink_path, symlink_path_sig)
break
#Abort if we couldn't create the symlink
if not symlink_created:
if symlink_path is None:
ka_print_error("Coudn't create upstream tarball symlink")
else:
ka_print_error("Coudn't create upstream tarball symlink in %s" % symlink_path)
ka_print_error("Tried the following paths to find the upstream tarball:")
for symlink_target_path in symlink_target_paths:
ka_print_error(" %s" % symlink_target_path)
sys.exit(1)
[docs]
def get_kci_tarball(self):
"""
Prepares a tarball which could be used to do a kci build.
"""
#Abort if the package is native
if self.is_native_package():
raise Exception("Trying to use get_kci_tarball() against a native package")
#Print info message
ka_print_plain("Getting kci tarball...")
#Read configuration files
ka_config = get_ka_config_parser()
#Make sure the build area directory exists, so we can store the resulting tarball
build_area = ka_config['areas']['build-area']
if not os.path.exists(build_area):
os.makedirs(build_area)
#Build the git url
upstream_name = self.upstream_git_name()
kde_git_repository_url = "https://anongit.kde.org/%s" % upstream_name
#Create temporary directory to clone the repository.
temp_dir = tempfile.TemporaryDirectory()
kde_clone_dir_name = os.path.join(temp_dir.name, upstream_name)
#Clone repository
git_command = ["git", "clone", kde_git_repository_url, kde_clone_dir_name]
ka_print_plain(" ".join(git_command))
returncode = subprocess.call(git_command)
if returncode != 0:
raise Exception("git clone failed")
#Remove .git directory
shutil.rmtree(kde_clone_dir_name + "/.git")
#Create the tarball
upstream_version = self.get_version().upstream_version
tar_command = ["tar", "cJf",
"%s/%s_%s.orig.tar.xz" % (build_area, upstream_name, upstream_version),
"-C", temp_dir.name,
upstream_name]
ka_print_plain(" ".join(tar_command))
returncode = subprocess.call(tar_command)
if returncode != 0:
raise Exception("tar execution failed")
[docs]
def unpack_upstream_source(self):
"""
Unpacks the upstream source code in the source package directory.
"""
#Abort if the package is native
if self.is_native_package():
raise Exception("Trying to use unpack_upstream_source() against a native package")
#Find existing tarball extension if possible
tarball_extension = None
symlink_target_paths = self.upstream_tarball_paths()
for symlink_target_path in symlink_target_paths:
if os.path.exists(symlink_target_path):
tarball_extension = get_orig_tarball_extension(symlink_target_path)
break
#Get the orig tarball if it doesn't exist already
if tarball_extension is None:
self.get_orig_tarball()
#Find out tarball extension again
for symlink_target_path in symlink_target_paths:
if os.path.exists(symlink_target_path):
tarball_extension = get_orig_tarball_extension(symlink_target_path)
break
#Find out the orig tarball path
orig_tarball_path = self.get_orig_tarball_path(tarball_extension)
#Get the tarball again if the symlink in build area doesn't exist
if not os.path.exists(orig_tarball_path):
self.get_orig_tarball()
#Try to extract the orig tarball
command = ["tar", "xf", orig_tarball_path, "--strip=1"]
ka_print_plain(" ".join(command))
try:
subprocess.check_call(command)
except Exception as exception: #pylint: disable=broad-except
ka_print_plain(str(exception))
sys.exit(1)
[docs]
def remove_broken_debian_breaks(self):
#TODO
pass
# pylint: disable=too-many-locals
[docs]
def sync_with_archive(self, dist, dist_version):
"""Executing this function will dowload the latest package from the Ubuntu/Debian archive
and it will copy its debian/* files to the current directory. The 'dist' parameter must be
either 'ubuntu' or 'debian'"""
#Check the dist parameter
if dist not in ['debian', 'ubuntu']:
raise Exception("Distribution %s is not either 'debian' or 'ubuntu'")
#Load changelog if needed
if not self._changelog_file_loaded:
self.load_changelog()
#Find out source package name
source_package_name = self.source_package_name()
#Login into Launchpad and get some data
lp_object = Launchpad.login_anonymously("Kubuntu Automation", "production")
lp_dist = lp_object.distributions[dist]
lp_archive = lp_dist.getArchive(name='primary')
lp_dist_version = lp_dist.getSeries(name_or_version=dist_version)
#Get the list of packages Pending or published.
status_to_check = ['Pending', 'Published']
pockets_to_check = ['Proposed', 'Updates', 'Security', 'Release']
lp_version = Version("0")
for pocket in pockets_to_check:
for status in status_to_check:
lp_packages = lp_archive.getPublishedSources(distro_series=lp_dist_version,
status=status,
pocket=pocket,
exact_match=True,
source_name=source_package_name)
for lp_package in lp_packages:
#Find out the archive version
package_version = Version(lp_package.source_package_version)
#Update version if it's higher
if package_version > lp_version:
lp_version = package_version
#Abort if the package wasn't found
if lp_version == Version("0"):
raise Exception("Package not found in the archive")
#Strip epoch from version
if lp_version.debian_revision:
source_package_version = (lp_version.upstream_version
+ '-' + lp_version.debian_revision)
else:
source_package_version = lp_version.upstream_version
#Find out upstream version
upstream_version = Version(source_package_version).upstream_version
#Find out the *.dsc url, so we will be able to dget it later
dsc_url_pattern = "https://launchpad.net/%s/+archive/primary/+files/%s_%s.dsc"
dsc_url = dsc_url_pattern % (dist, source_package_name, source_package_version)
#Create the temporary directory to download and extract the source package
temp_dir = tempfile.TemporaryDirectory()
#Move to the temporary directory and remember the current cwd
old_cwd = os.getcwd()
os.chdir(temp_dir.name)
#Download the source package
dget_command = "dget -x -u %s" % dsc_url
ka_print_plain(dget_command)
return_code = os.system(dget_command)
if return_code != 0:
raise Exception("Download of %s failed." % dsc_url)
#Go back to the old CWD
os.chdir(old_cwd)
#Find out source and destination directories for rsync call below
downloaded_source_dir = os.path.join(temp_dir.name,
source_package_name
+ '-' + upstream_version)
if not self.is_native_package():
rsync_source_dir = os.path.join(downloaded_source_dir, 'debian')
rsync_dest_dir = os.path.join(old_cwd, 'debian')
else:
rsync_source_dir = downloaded_source_dir
rsync_dest_dir = old_cwd
#Copy the entire source package in the current directory with rsync
ka_print_good_stuff("Copying source package files... ")
sys.stdout.flush()
os.system("rsync -avz --delete-before "
"--exclude=.git* " # this prevents the rsync deleting our local git clone
"%s/ %s/" % (rsync_source_dir, rsync_dest_dir))
if return_code != 0:
raise Exception("Copy of the source package failed.")
sys.stdout.flush()
ka_print_good_stuff("DONE")
sys.stdout.flush()
# pylint: enable=too-many-locals
def _get_ppa_suffix_number(self, dist):
#Get source package name
src_pkg_name = self.source_package_name()
#Find out information about the appropiate staging ppa for this package
kppa = KubuntuPPA(self.get_release_type())
#Login into launchpad to get the latest version of the package
try:
#Login into Launchpad and get some data
lp_object = Launchpad.login_anonymously("Kubuntu Automation", "production")
ppa = lp_object.people[kppa.get_user_name()].getPPAByName(name=kppa.get_ppa_name())
lp_dist = lp_object.distributions["ubuntu"].getSeries(name_or_version=dist)
#Get the list of packages Pending or published.
lp_packages = ppa.getPublishedSources(distro_series=lp_dist, status='Pending',
exact_match=True, source_name=src_pkg_name)
if not lp_packages:
lp_packages = ppa.getPublishedSources(distro_series=lp_dist, status='Published',
exact_match=True, source_name=src_pkg_name)
#Increase the version suffix
if len(lp_packages) >= 1:
#Find the package with the higher version in the PPA
lp_package = lp_packages[0]
lp_package_version = Version(lp_package.source_package_version)
for tmp_lp_pkg in lp_packages:
tmp_lp_pkg_version = Version(tmp_lp_pkg.source_package_version)
if tmp_lp_pkg_version > lp_package_version:
lp_package = tmp_lp_pkg
lp_package_version = tmp_lp_pkg_version
#The package was already uploaded in the staging PPA
lp_version = lp_package.source_package_version
local_version = self.get_version()
if re.search('^' + re.escape(str(local_version)), lp_version) is not None:
#The package was already uploaded having the same version
#(except for the suffix)
suffix_number_match = re.search(r'\d+$', lp_version)
if suffix_number_match is not None:
suffix_number = str(int(suffix_number_match.group(0))+1)
else:
#The package was already uploaded but the version differs more
#than just the suffix, so lets start with suffix 1
suffix_number = "1"
else:
suffix_number = "1"
#Return result
return suffix_number
except Exception as exception: # pylint: disable=broad-except
ka_print_plain(str(exception))
ka_print_error("PPA suffix guessing failed. "
"You can override the suffix with '-s <suffix>'")
sys.exit(1)
[docs]
def get_ppa_suffix(self, dist, suffix_number):
"""
Returns a PPA suffix for the current package.
"""
if not self._changelog_file_loaded:
self.load_changelog()
if suffix_number is None:
suffix_number = self._get_ppa_suffix_number(dist)
return self._changelog.get_ppa_suffix(dist, suffix_number)
[docs]
def add_ppa_suffix(self, dist, suffix_number):
"""
Adds a PPA suffix to the changelog.
"""
if not self._changelog_file_loaded:
self.load_changelog()
if suffix_number is None:
suffix_number = self._get_ppa_suffix_number(dist)
self._changelog.add_ppa_suffix(dist, suffix_number)
self._changelog.dump()
[docs]
def get_qt_ppa_suffix(self, dist, end_number, begin_number=0):
"""
Returns a Qt PPA suffix for the current package.
"""
if not self._changelog_file_loaded:
self.load_changelog()
if end_number is None:
end_number = self._get_ppa_suffix_number(dist)
return self._changelog.get_qt_ppa_suffix(dist, end_number, begin_number)
[docs]
def add_qt_ppa_suffix(self, dist, end_number, begin_number=0):
"""
Adds a Qt PPA suffix to the changelog.
"""
if not self._changelog_file_loaded:
self.load_changelog()
if end_number is None:
end_number = self._get_ppa_suffix_number(dist)
self._changelog.add_qt_ppa_suffix(dist, end_number, begin_number)
self._changelog.dump()
#pylint: disable=unused-argument
def _get_tritemio_suffix_number(self, dist):
#Find out source package name
src_pkg_name = self.source_package_name()
#Get the package information at tritemio's repository
reprepro_reply = tritemio_ls(src_pkg_name, arch='source').splitlines()
#Find out the suffix number: it will be the higher@reprepro+1
suffix_number = 1
local_version = self.get_version()
for line in reprepro_reply:
version_line = line.split('|')[1].strip()
if re.search('^' + str(local_version), version_line) is not None:
#The package was already uploaded having the same version but different suffix
suffix_number_match = re.search(r'\d+$', version_line)
if suffix_number_match is not None:
suffix_number_candidate = int(suffix_number_match.group(0))+1
#If the suffix candidate is greater, let's just update it
if suffix_number_candidate > suffix_number:
suffix_number = suffix_number_candidate
#Return result
return suffix_number
#pylint: enable=unused-argument
[docs]
def get_tritemio_suffix(self, dist, suffix_number, suffix_offset=0):
"""
Returns a tritemio suffix for the current package.
"""
if not self._changelog_file_loaded:
self.load_changelog()
if suffix_number is None:
suffix_number = self._get_tritemio_suffix_number(dist)
suffix_number = int(suffix_number) + int(suffix_offset)
return self._changelog.get_tritemio_suffix(dist, suffix_number)
[docs]
def add_tritemio_suffix(self, dist, suffix_number, suffix_offset=0):
"""
Add a tritemio suffix to the changelog.
"""
if not self._changelog_file_loaded:
self.load_changelog()
if suffix_number is None:
suffix_number = self._get_tritemio_suffix_number(dist)
suffix_number = int(suffix_number) + int(suffix_offset)
self._changelog.add_tritemio_suffix(dist, suffix_number)
self._changelog.dump()
[docs]
def get_stability(self, version):
"""
Returns a string naming the type of release ("beta", RC", ...) for the
given version.
"""
#Get relevant data about the package
version = str(version)
release_type = self.get_release_type()
#Find out stability
stability = None
if release_type == "plasma":
if version.endswith(".90"):
stability = "beta"
elif release_type == "applications":
if version.endswith(".80"):
stability = "beta"
elif version.endswith(".90"):
stability = "RC"
#Return result
return stability
[docs]
def bump_build_depends(self, dist, release_type=None, bump_autopkgtest=False):
"""
Bumps all build depends using ka-metadata.
You may use the `release_type` parameter to force the "release type" (i.e "frameworks",
"plasma", "applications" ...). If it's `None` it will be automatically calculated.
If `bump_autopkgtest` is `True`, it will bump the autopkgtest Depends in
'debian/tests/control'.
"""
#Load control file if needed
if not self._control_file_loaded:
self.load_control_file()
#Find out release type
if release_type is None:
release_type = self.get_release_type()
#Find out build depends map
build_depends_map = read_json_data_file("dev-package-name-lists/%s-%s.json"
% (release_type, dist))
#Open the overrides file if exists
overrides_file = "dev-package-name-lists/%s-%s-overrides.json" % (release_type, dist)
try:
overrides_map = read_json_data_file(overrides_file, catch_exceptions=False)
except FileNotFoundError:
overrides_map = {}
#Add the overrides to the build depends map
build_depends_map.update(overrides_map)
#Bump debian/control Build-Depends[-Indep]
self._control_file.bump_versions_with_map(':source:',
'Build-Depends', build_depends_map)
self._control_file.bump_versions_with_map(':source:',
'Build-Depends-Indep', build_depends_map)
self._control_file.wrap_and_sort_field(':source:', 'Build-Depends')
self._control_file.wrap_and_sort_field(':source:', 'Build-Depends-Indep')
#Bump the -dev depends
bin_pkg_names = self._control_file.get_bin_packages()
for bin_pkg_name in bin_pkg_names:
if bin_pkg_name in build_depends_map:
self._control_file.bump_versions_with_map(bin_pkg_name,
'Depends', build_depends_map)
self._control_file.wrap_and_sort_field(bin_pkg_name, 'Depends')
#Dump debian/control contents to hard disk
self._control_file.dump()
#Bump the autopkgtest Depends if required
if bump_autopkgtest:
#Read the file
if not self._tests_control_file_loaded:
self.load_tests_control_file()
#Abort and do nothing if there's no debian/tests/control
if self._tests_control_file is None:
return
#Find out binary packages map
bin_pkgs_map = read_json_data_file("bin-package-name-lists/%s-%s.json"
% (release_type, dist))
#Find out test names
test_names = self._tests_control_file.get_bin_packages()
#Bump the 'Depends' for each test
for test_name in test_names:
self._tests_control_file.bump_versions_with_map(test_name,
'Depends', bin_pkgs_map)
#Dump debian/tests/control file to hard disk
self._tests_control_file.dump()
[docs]
def replace_build_depend(self, old_bd, new_bd, control_field='Build-Depends'):
r"""
Replaces the package given in `old_bd` with `new_bd` in the Build-Depends field or
in the field given in `control_field`.
If `old_bd` is not found, this method will return `False`, otherwise it will return
`True`.
"""
#Load control file if needed
if not self._control_file_loaded:
self.load_control_file()
#Remove the old build depend if possible
bd_found = self._control_file.remove_from_relation(':source:', control_field, old_bd)
#Add the new build depend if the old one was found
if bd_found:
self._control_file.add_to_relation(':source:', control_field, new_bd)
#Put the field in wrap-and-sort style
self._control_file.wrap_and_sort_field(':source:', control_field)
#Dump debian/control contents to hard disk
self._control_file.dump()
#Return function result
return bd_found
#pylint: disable=too-many-branches
[docs]
def new_upstream_release(self, dist, new_upstream_version=None, bump_autopkgtest=False):
"""
Makes all the needed changes in the packaging to provide a new upstream release.
If the new upstream release was already done, returns False and does nothing, otherwise
returns the new upstream version.
If `bump_autopkgtest` is `True`, it will bump the autopkgtest Depends in
'debian/tests/control'.
"""
#Load changelog if needed
if not self._changelog_file_loaded:
self.load_changelog()
#Find out release_type
release_type = self.get_release_type()
#Find out source package name
source_package_name = self.source_package_name()
if release_type == "qt":
if source_package_name.endswith('opensource-src'):
source_package_name = source_package_name.split('-opensource-src')[0]
source_package_name += '-everywhere-src'
#Check the release type to do whatever is appropiate
if release_type in ["qt", "frameworks", "plasma", "applications"]:
#Find out upstream tarball name
upstream_tarball_name = self.upstream_tarball_name()
#Find out the new upstream version if needed
if new_upstream_version is None:
real_release_type = self.get_release_type(use_overrides=False)
new_upstream_version = get_ftp_version_map(real_release_type)[upstream_tarball_name]
#Abort in the latest changelog block already provides the new upstream release
last_changelog_upstream_version = self._changelog.get_version().upstream_version
if last_changelog_upstream_version.startswith(new_upstream_version):
return False
#Bump build depends
self.bump_build_depends(dist, None, bump_autopkgtest)
else:
if new_upstream_version is None:
raise Exception("If the release type of the package is not "
"'qt', 'frameworks', 'plasma' or 'applications' "
"you must pick a new_upstream_version")
#Abort in the latest changelog block already provides the new upstream release
last_changelog_upstream_version = self._changelog.get_version().upstream_version
if last_changelog_upstream_version.startswith(new_upstream_version):
return False
#Prepare changelog
stability = self.get_stability(new_upstream_version)
self._changelog.new_upstream_release(new_upstream_version, stability)
self._changelog.dump()
#If we reached this point we actually updated the package, so return True
return new_upstream_version
#pylint: enable=too-many-branches
[docs]
def set_kubuntu_maintainer_fields(self):
"""
Sets the maintainer fields for Kubuntu.
Returns `True` if the file was actually changed.
"""
#Load control file if not loaded already
if not self._control_file_loaded:
self.load_control_file()
#Set the fields in debian/control and return result
control_file_changed = self._control_file.set_kubuntu_maintainer_fields()
if control_file_changed:
self._control_file.dump()
return control_file_changed
[docs]
def set_kubuntu_vcs_fields(self):
"""
Sets the vcs fields for Kubuntu.
Returns `True` if the file was actually changed.
"""
#Load control and watch files if not loaded already
if not self._control_file_loaded:
self.load_control_file()
if not self._watch_file_loaded:
self.load_watch_file()
#Find out source package name
src_pkg_name = self.source_package_name()
#Find out release type
release_type = self.get_release_type()
#Find out repo name
if self.is_native_package():
repo_name = self.source_package_name()
else:
repo_name = self._watch_file.tarball_name(src_pkg_name)
#Set the fields in debian/control and return result
control_file_changed = self._control_file.set_kubuntu_vcs_fields(release_type, repo_name)
if control_file_changed:
self._control_file.dump()
return control_file_changed
[docs]
def is_part_of_qt_bootstrap_set(self, dist):
"""
This method return `True` if the package is part of the Qt bootstrap set
configured in `ka-metadata/qt-bootstrap/<dist>`
"""
qt_bootstrap_set = read_packages_file('qt-bootstrap/%s' % dist)
mangled_package_name = self.source_package_name().split('-everywhere-src')[0]
mangled_package_name = mangled_package_name.split('-opensource-src')[0]
return mangled_package_name in qt_bootstrap_set
[docs]
def prepare_qt_stage1(self, dist):
"""
This method prepares the packaging for the first stage of Qt bootstraping.
It checks if the package is part of the bootstrap set checking the
`ka-metadata/qt-bootstrap/<dist>` file; if the package is part to that set,
this method will apply the needed changes and it will return `True`; if it's
not part of the bootstrap set it will do nothing and return False.
"""
#Abort if the package is not part of the bootstrap set
if not self.is_part_of_qt_bootstrap_set(dist):
return False
#Remove the doc Build-Depends[-Indep]
if not self._control_file_loaded:
self.load_control_file()
self._control_file.remove_doc_build_depends()
self._control_file.dump()
#Alter debian/rules to build in 'nodoc' mode
rules_lines = []
line_count = 0
rules_file = open('debian/rules', 'r')
for line in rules_file:
line_count += 1
rules_lines.append(line)
if line_count == 2:
rules_lines.append("\n#Build without docs for bootstrap stage 1\n"
"export DEB_BUILD_PROFILES=nodoc\n\n")
rules_file.close()
rules_file = open('debian/rules', 'w')
for line in rules_lines:
rules_file.write(line)
rules_file.close()
#If we reached this point we actually modified the package, so return True
return True
[docs]
def has_autopkgtests_disabled(self): #pylint: disable=no-self-use
"""
Returns `True` if this package has autopktests there, but disabled.
"""
return os.path.isfile('debian/tests/control.disabled')
[docs]
def restore_autopkgtests(self):
"""
Tries to restore autopkgtests if they are available but disabled.
Returns `True` if they were indeed re-enabled.
"""
must_restore = self.has_autopkgtests_disabled()
if must_restore:
os.rename('debian/tests/control.disabled', 'debian/tests/control')
return must_restore