diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 0ccfd2fef..83b4351ab 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -593,6 +593,7 @@ exit 0 %files pkglistgen %{_bindir}/osrt-pkglistgen +%{_bindir}/osrt-git-pkglistgen %{_bindir}/osrt-skippkg-finder %{_datadir}/%{source_dir}/pkglistgen %{_datadir}/%{source_dir}/pkglistgen.py diff --git a/git-pkglistgen.py b/git-pkglistgen.py new file mode 100755 index 000000000..c9d109b49 --- /dev/null +++ b/git-pkglistgen.py @@ -0,0 +1,401 @@ +#!/usr/bin/python3 +import os +import sys +import ReviewBot + +import logging + +import traceback + +import re + +import subprocess + +import tempfile + +from urllib.parse import urljoin, urldefrag +import urllib.error + +from lxml import etree + +from osc import conf +from osc.core import makeurl, make_meta_url, http_POST, http_PUT +from osclib.conf import Config +from osclib.stagingapi import StagingAPI +from pkglistgen.engine import Engine +from pkglistgen.tool import PkgListGen, MismatchedRepoException + +from collections import namedtuple + +DEFAULT_AUTOGITS_REVIEWER = "autogits_obs_staging_bot" +DEFAULT_ENGINE = "product_composer" +DEFAULT_ENABLE_REPOSITORIES = "product images" + +STAGING_PROGRESS_MARKER = "staging/In Progress" +STAGING_TYPE_MARKERS = "QA-SLES-Basic QA-SLES-Reduced QA-SLES-Full" + +slugify_regex = re.compile("[^a-z0-9_]+") + +StagingProject = namedtuple("StagingProject", ["target", "name", "origin", "label"]) + + +def slugify(x): + return slugify_regex.sub("-", x.lower()) + + +class GitRepository(object): + + def __init__(self, origin_remote): + + self.origin_remote = origin_remote + + # This gets cleaned up on exit + self.temporary_directory = tempfile.TemporaryDirectory(suffix="pkglistgen") + + self.git_checkout = os.path.join(self.temporary_directory.name, "git") + + def fetch(self): + if not os.path.exists(self.git_checkout): + subprocess.check_call( + ["git", "clone", "--mirror", self.origin_remote, self.git_checkout] + ) + + # Fetch + subprocess.check_call( + ["git", "fetch", self.origin_remote], cwd=self.git_checkout + ) + + def push_to_branch(self, source_pointer, target_remote, target_branch): + + if ( + subprocess.run( + ["git", "rev-parse", "--verify", f"refs/heads/{source_pointer}"], + cwd=self.git_checkout, + ).returncode + > 0 + ): + # commit/tag + source_ref = source_pointer + else: + # branch + source_ref = f"refs/heads/{source_pointer}" + + subprocess.check_call( + [ + "git", + "push", + target_remote, + f"{source_ref}:refs/heads/{target_branch}", + ], + cwd=self.git_checkout, + ) + + +class GitRepositories(object): + + def __init__(self): + self.mapping = {} + + def __getitem__(self, origin_remote): + if origin_remote not in self.mapping: + self.mapping[origin_remote] = GitRepository(origin_remote) + + return self.mapping[origin_remote] + + +class GitPkgListGenBot(ReviewBot.ReviewBot): + """A review bot that runs pkglistgen on staging QA projects""" + + def __init__(self, *args, **kwargs): + ReviewBot.ReviewBot.__init__(self, *args, **kwargs) + + conf.get_config() + + self.tool = PkgListGen() + self.apiurl = conf.config["apiurl"] + + self.allowed_repositories = [] + + self.staging_origin_cache = {} + + self.cloned_repositories = GitRepositories() + + # This is heavily dependent on the GITEA platform + if self.platform.name != "GITEA": + raise Exception("Unsupported platform: this bot is only supported on Gitea") + + def get_git_staging_configuration(self, owner, project, commit_sha): + # FIXME: support JWCC + return self.platform.get_path( + f"repos/{owner}/{project}/raw/staging.config?ref={commit_sha}" + ).json() + + def get_qa_projects(self, request_id, staging_configuration): + base_project = staging_configuration["StagingProject"] + for project in staging_configuration.get("QA", []): + yield StagingProject( + target=f"{base_project}:{request_id}:{project['Name']}", + origin=project["Origin"], + name=project["Name"], + label=project.get("Label"), + ) + + @staticmethod + def is_request_approved_by(request, approver): + for review in request.reviews: + if review.by == approver and review.state == "accepted": + # We skip dismissed reviews, so we can afford returning + # as soon as we find a matching review + return True + + return False + + @staticmethod + def get_request_from_src_rev(requests, src_rev): + for request in requests: + if request.actions[0].src_rev == src_rev: + return request + + return None + + def set_project_flag(self, project, flag, repository, status): + return http_POST( + makeurl( + self.apiurl, + ["source", project], + { + "cmd": "set_flag", + "flag": flag, + "repository": repository, + "status": status, + }, + ) + ) + + def replace_meta(self, project, meta_element: etree.ElementTree): + return http_PUT( + make_meta_url("prj", project, self.apiurl), + data=etree.tostring(meta_element, encoding="utf-8", xml_declaration=True), + ) + + def check_source_submission( + self, src_owner, src_project, src_rev, target_owner, target_package + ): + self.logger.info(f"Checking {src_project}: {src_owner} -> {target_owner}") + + try: + result, message = self.run_pkglistgen( + src_owner, src_project, src_rev, target_owner, target_package + ) + except Exception: + self.review_messages["declined"] = ( + f"Unhandled exception:\n\n```{traceback.format_exc()}```" + ) + return False + + if result: + self.review_messages["accepted"] = message or "OK" + + return result # True or None + + def run_pkglistgen( + self, src_owner, src_project, src_rev, target_owner, target_package + ): + """ + Runs pkglistgen. + + :return: result: True (pkglistgen ran), or None (should skip/retry later). + message: a message that should be shown into the comment, or None. + """ + + request = self.get_request_from_src_rev(self.requests, src_rev) + if not request: + self.logger.warning(f"Request for src_rev {src_rev} not found") + return None, None + + if f"{request._owner}/{request._repo}" not in self.allowed_repositories: + self.logger.info( + f"{request._owner}/{request._repo} is not in the allowed repositories list" + ) + return None, None + + if STAGING_PROGRESS_MARKER not in request._labels: + self.logger.info( + f"PR {request._owner}/{request._repo}#{request._pr_id} is not in progress" + ) + return None, None + + base_commit = request.actions[0].tgt_rev + staging_configuration = self.get_git_staging_configuration( + target_owner, target_package, base_commit + ) + + if "QA" not in staging_configuration: + self.logger.warning( + f"PR {request._owner}/{request._repo}#{request._pr_id} has no QA staging configured" + ) + return None, None + + main_project = staging_configuration["ObsProject"] + + Config(self.apiurl, main_project) + target_config = conf.config[main_project] + + main_repo = target_config["main-repo"] + staging_org_url = target_config["pkglistgen-git-staging-org-url"] + if not staging_org_url.endswith("/"): + staging_org_url += "/" + staging_branch = slugify( + f"qa_{request._owner}_{request._repo}_pr{request._pr_id}" + ) + engine = Engine[target_config.get("pkglistgen-engine", DEFAULT_ENGINE)] + enable_repositories = target_config.get( + "pkglistgen-enable-repositories", DEFAULT_ENABLE_REPOSITORIES + ).split(" ") + staging_type_markers = target_config.get( + "staging-type-markers", STAGING_TYPE_MARKERS + ).split(" ") + + approver = target_config.get("pkglistgen-approver", DEFAULT_AUTOGITS_REVIEWER) + if not self.is_request_approved_by(request, approver): + return None, None + + if True not in [x in staging_type_markers for x in request._labels]: + self.logger.info( + f"PR {request._owner}/{request._repo}#{request._pr_id} has no valid staging markers set" + ) + return True, "Not asked to create stagings. Accepting." + + staging_project_available = False + for qa_project in self.get_qa_projects(request._pr_id, staging_configuration): + api = StagingAPI(self.apiurl, qa_project.target) + + try: + meta = api.get_prj_meta(qa_project.target) + except urllib.error.HTTPError as e: + if e.code == 404: + # Let's go ahead, as the QA project might have been masked by the used Label, + # and we will check that further on + continue + else: + raise + else: + staging_project_available = True + + # Obtain the target repository name by looking at the repository name + # in the Origin's scmsync. We cannot use qa_project["Name"] in this case + # as after labels have been introduced, the same origin might have multiple + # QA projects + if qa_project.origin not in self.staging_origin_cache: + origin_meta = api.get_prj_meta(qa_project.origin) + origin_git_url_element = origin_meta.xpath("/project/scmsync")[0] + + url, fragment = urldefrag(origin_git_url_element.text) + self.staging_origin_cache[qa_project.origin] = url.split("/")[ + -1 + ].replace(".git", "") + + staging_repo_url = urljoin( + staging_org_url, self.staging_origin_cache[qa_project.origin] + ) + target_git_url = urljoin(staging_repo_url, f"#{staging_branch}") + git_url_element = meta.xpath("/project/scmsync")[0] + + if not git_url_element.text.startswith( + "http" + ) or not target_git_url.startswith("http"): + # We do not expect nor support non-http[s] uris + raise Exception("Only http[s] git remote uris are supported") + + if git_url_element.text != target_git_url and not self.dryrun: + # Should do the initial push + url, fragment = urldefrag(git_url_element.text) + self.logger.info(f"Creating branch {staging_branch}") + self.cloned_repositories[url].fetch() + self.cloned_repositories[url].push_to_branch( + fragment, staging_repo_url, staging_branch + ) + + git_url_element.text = target_git_url + + self.replace_meta(qa_project.target, meta) + + # We will get back to it later + return None, None + + self.tool.reset() + self.tool.dry_run = self.dryrun + try: + self.tool.update_and_solve_target( + api, + main_project, + target_config, + main_repo, + git_url=git_url_element.text, + project=qa_project.target, + scope="target", + engine=engine, + force=True, + no_checkout=False, + only_release_packages=False, + only_update_weakremovers=False, + stop_after_solve=False, + custom_cache_tag="git-pkglistgen", + ) + except MismatchedRepoException: + # Repo still building, just exit now as presumably eventual + # other projects are also affected + self.logger.warning("Repository is still building, trying next time...") + return None, None + else: + # Enable builds + if not self.dryrun: + for repository in enable_repositories: + self.set_project_flag( + qa_project.target, "build", repository, "enable" + ) + + if staging_project_available: + return True, "pkglistgen ran successfully" + else: + self.logger.info( + "Staging bot didn't create the QA project, but accepted the review. Nothing to do." + ) + return ( + True, + "Staging bot didn't create the QA project, but accepted the review. Nothing to do.", + ) + + +class CommandLineInterface(ReviewBot.CommandLineInterface): + def __init__(self, *args, **kwargs): + super().__init__(*args, *kwargs) + self.clazz = GitPkgListGenBot + + def get_optparser(self): + parser = super().get_optparser() + + # Add bot-specific options + # If ReviewBot/Cmdln moves to ArgumentParser, we can turn this into a + # string directly and use nargs=*. + parser.add_option( + "--git-allow-repos", + default="", + help="allowed git repositories (e.g. products/SLFO,products/SLES)", + ) + + return parser + + def setup_checker(self): + instance = super().setup_checker() + + instance.allowed_repositories = self.options.git_allow_repos.split(",") + + return instance + + +if __name__ == "__main__": + app = CommandLineInterface() + logging.basicConfig(level=logging.DEBUG) + + sys.exit(app.main()) diff --git a/pkglistgen/tool.py b/pkglistgen/tool.py index c724a2903..a71a99b19 100644 --- a/pkglistgen/tool.py +++ b/pkglistgen/tool.py @@ -927,11 +927,18 @@ def update_and_solve_target( ['git', 'add', self.output_dir], cwd=cache_dir, encoding='utf-8')) - logging.debug(subprocess.check_output( - ['git', 'commit', '-m', 'Update by pkglistgen of openSUSE-release-tool'], cwd=cache_dir, encoding='utf-8')) - if not self.dry_run: + # Check if any changes are staged (i.e. they have been caught + # up by the git call above) + # Returncode 1 means that there is something staged, so that + # we can actually commit + if subprocess.run(['git', 'diff', '--cached', '--stat', '--quiet'], cwd=cache_dir).returncode == 1: logging.debug(subprocess.check_output( - ['git', 'push'], cwd=cache_dir)) + ['git', 'commit', '-m', 'Update by pkglistgen of openSUSE-release-tool'], cwd=cache_dir, encoding='utf-8')) + if not self.dry_run: + logging.debug(subprocess.check_output( + ['git', 'push'], cwd=cache_dir)) + else: + logging.debug("No changes to commit.") elif not self.dry_run: self.commit_package(self.output_dir) diff --git a/plat/gitea.py b/plat/gitea.py index 21e2c7096..3c246e268 100644 --- a/plat/gitea.py +++ b/plat/gitea.py @@ -168,6 +168,25 @@ def __init__(self, type, json): self._set_attr_from_json('tgt_rev', json, 'base.sha') +class Review: + attributes = ["by", "state", "type", "when"] + + states_mapping = { + "APPROVED": "accepted", + "REQUEST_CHANGES": "declined", + "REQUEST_REVIEW": "new", + } + + def __init__(self, **kwargs): + self._review_data = kwargs + + def __getattr__(self, attribute): + if attribute == "state": + return self.states_mapping[self._review_data.get("state", "REQUEST_REVIEW")] + + return self._review_data.get(attribute, None) + + class Request: """Request structure implemented for Gitea""" def __init__(self): @@ -182,6 +201,25 @@ def parse_request_id(reqid): def construct_request_id(owner, repo, pr_id): return f'{owner}:{repo}:{pr_id}' + @staticmethod + def format_review(review): + if review.get("user") is not None: + return Review( + by=review["user"]["login"], + state=review["state"], + type="User", + when=review["updated_at"] + ) + elif review.get("team") is not None: + return Review( + by=review["team"]["name"], + state=review["state"], + type="Group", + when=review["updated_at"] + ) + else: + raise Exception("Unknown review type") + def _init_attributes(self): self.reqid = None self.creator = '' @@ -197,30 +235,37 @@ def _init_attributes(self): self._issues = None # Gitea specific attributes + self._labels = set() self._owner = None self._repo = None self._pr_id = None - def read(self, json, owner, repo): + def read(self, request_json, reviews_json, owner, repo): """Read in a request from JSON response""" self._init_attributes() self._owner = owner self._repo = repo - self._pr_id = json["number"] + self._pr_id = request_json["number"] - self.reqid = Request.construct_request_id(owner, repo, json["number"]) - self.creator = json["user"]["login"] - self.created_at = json["created_at"] - self.updated_at = json["updated_at"] - self.title = json["title"] - self.description = json["body"] - self.state = json["state"] + self.reqid = Request.construct_request_id(owner, repo, request_json["number"]) + self.creator = request_json["user"]["login"] + self.created_at = request_json["created_at"] + self.updated_at = request_json["updated_at"] + self.title = request_json["title"] + self.description = request_json["body"] + self.state = request_json["state"] - self.actions = [RequestAction(type="submit", json=json)] + self.actions = [RequestAction(type="submit", json=request_json)] - if json.get("merged"): - self.accept_at = json["merged_at"] + if request_json.get("merged"): + self.accept_at = request_json["merged_at"] + + for review in reviews_json: + if not review.get("dismissed", False): + self.reviews.append(Request.format_review(review)) + + self._labels = {x["name"] for x in request_json["labels"]} class ProjectConfig: @@ -250,8 +295,10 @@ def get_path(self, *args): def _get_request(self, pr_id, owner, repo): res = self.api.get(f'repos/{owner}/{repo}/pulls/{pr_id}').json() + reviews = self.api.get(f'repos/{owner}/{repo}/pulls/{pr_id}/reviews').json() + ret = Request() - ret.read(res, owner=owner, repo=repo) + ret.read(res, reviews, owner=owner, repo=repo) return ret def get_request(self, request_id, with_full_history=False):