From b08008aba683afc25340aff797622c771c8cf49e Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sat, 31 Jan 2026 02:20:09 +0100 Subject: [PATCH 1/2] Add command to trigger reticulum build with version overrides --- lib/pyxis/commands/components.rb | 22 ++++++ lib/pyxis/github_client.rb | 18 +++-- lib/pyxis/project/base.rb | 4 + .../create_reticulum_build_service.rb | 73 +++++++++++++++++++ 4 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 lib/pyxis/services/create_reticulum_build_service.rb diff --git a/lib/pyxis/commands/components.rb b/lib/pyxis/commands/components.rb index c9abe54..b42d843 100644 --- a/lib/pyxis/commands/components.rb +++ b/lib/pyxis/commands/components.rb @@ -19,6 +19,28 @@ def info result end + desc 'build', 'Start a reticulum build with specific versions' + method_option :aquila_sha, desc: 'Commit SHA of aquila to build', type: :string + method_option :draco_sha, desc: 'Commit SHA of draco to build', type: :string + method_option :sagittarius_sha, desc: 'Commit SHA of sagittarius to build', type: :string + method_option :sculptor_sha, desc: 'Commit SHA of sculptor to build', type: :string + method_option :taurus_sha, desc: 'Commit SHA of taurus to build', type: :string + def build + version_overrides = { + aquila: options[:aquila_sha], + draco: options[:draco_sha], + sagittarius: options[:sagittarius_sha], + sculptor: options[:sculptor_sha], + taurus: options[:taurus_sha], + }.compact + + pipeline = Pyxis::Services::CreateReticulumBuildService.new(version_overrides).execute + + raise Pyxis::MessageError, 'Failed to create pipeline' if pipeline.nil? + + "Created reticulum build at #{pipeline.web_url}" + end + desc 'update', 'Update a component in reticulum' method_option :component, aliases: '-c', desc: 'The component to update', required: true, type: :string def update diff --git a/lib/pyxis/github_client.rb b/lib/pyxis/github_client.rb index f01b539..6166446 100644 --- a/lib/pyxis/github_client.rb +++ b/lib/pyxis/github_client.rb @@ -31,18 +31,26 @@ def without_auto_pagination(octokit) octokit.instance_variable_set(:@auto_paginate, current_auto_paginate) end - private - - def create_octokit(instance) - logger.info('Creating octokit client', instance: instance) + def create_installation_access_token(instance, options = {}) config = CLIENT_CONFIGS[instance] global_client = Octokit::Client.new(bearer_token: create_jwt(config[:private_key_location], config[:app_id])) logger.debug('Created JWT for client', app: global_client.app.slug) installation_token = Pyxis::GlobalStatus.with_faraday_dry_run_bypass do - global_client.create_app_installation_access_token(config[:installation_id]) + global_client.create_app_installation_access_token(config[:installation_id], options) end + logger.debug('Created app installation access token', installation_token: installation_token.to_h.except(:token)) + + installation_token + end + + private + + def create_octokit(instance) + logger.info('Creating octokit client', instance: instance) + + installation_token = create_installation_access_token(instance) Octokit::Client.new(bearer_token: installation_token[:token]) end diff --git a/lib/pyxis/project/base.rb b/lib/pyxis/project/base.rb index 7c55116..d45d609 100644 --- a/lib/pyxis/project/base.rb +++ b/lib/pyxis/project/base.rb @@ -16,6 +16,10 @@ def github_path paths[:github] end + def github_repository_name + github_path.split('/')[1] + end + def api_gitlab_path paths[:gitlab].gsub('/', '%2F') end diff --git a/lib/pyxis/services/create_reticulum_build_service.rb b/lib/pyxis/services/create_reticulum_build_service.rb new file mode 100644 index 0000000..2f1de74 --- /dev/null +++ b/lib/pyxis/services/create_reticulum_build_service.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Pyxis + module Services + class CreateReticulumBuildService + InvalidVersionOverride = Class.new(Pyxis::Error) + + include SemanticLogger::Loggable + + attr_reader :version_overrides + + def initialize(version_overrides) + @version_overrides = version_overrides + end + + def execute + logger.info('Creating build with version overrides', version_overrides: version_overrides) + + version_overrides.each_pair do |component, version| + validate_override!(component, version) + end + + pipeline = GitlabClient.client.post_json( + "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipeline", + { + ref: Project::Reticulum.default_branch, + variables: version_override_variables + token_variable, + } + ) + + pipeline.body if pipeline.response.status == 201 + end + + private + + def validate_override!(component, version) + project = Pyxis::Project.const_get(component.capitalize) + + begin + GithubClient.octokit.tag(project.github_path, version) + rescue Octokit::UnprocessableEntity, Octokit::NotFound + begin + GithubClient.octokit.commit(project.github_path, version) + rescue Octokit::UnprocessableEntity, Octokit::NotFound + raise InvalidVersionOverride, "Invalid version '#{version}' for component '#{component}'" + end + end + end + + def version_override_variables + variables = [] + + version_overrides.each_pair do |component, version| + variables << { + key: "OVERRIDE_#{component}_VERSION", + value: version, + } + end + + variables + end + + def token_variable + [ + { + key: 'C0_GH_TOKEN', + value: File.read(ENV.fetch('PYXIS_GH_RETICULUM_PUBLISH_TOKEN')), + } + ] + end + end + end +end From fb91d2d756c1330275543299e340394e2dc1b2c3 Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sat, 31 Jan 2026 20:57:23 +0100 Subject: [PATCH 2/2] Update component info command to use new image annotations --- lib/pyxis/dry_run_enforcer.rb | 2 +- lib/pyxis/generic_faraday.rb | 33 +++++++ lib/pyxis/gitlab_client.rb | 58 ++++++++----- .../managed_versioning/component_info.rb | 86 +++++++++++++++---- 4 files changed, 141 insertions(+), 38 deletions(-) create mode 100644 lib/pyxis/generic_faraday.rb diff --git a/lib/pyxis/dry_run_enforcer.rb b/lib/pyxis/dry_run_enforcer.rb index 1d7aa12..5744453 100644 --- a/lib/pyxis/dry_run_enforcer.rb +++ b/lib/pyxis/dry_run_enforcer.rb @@ -3,7 +3,7 @@ module Pyxis module DryRunEnforcer class FaradayBlocker < Faraday::Middleware - DryRunError = Class.new(StandardError) + DryRunError = Class.new(Pyxis::Error) include ::SemanticLogger::Loggable def on_request(env) diff --git a/lib/pyxis/generic_faraday.rb b/lib/pyxis/generic_faraday.rb new file mode 100644 index 0000000..ac5554d --- /dev/null +++ b/lib/pyxis/generic_faraday.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Pyxis + class GenericFaraday + include SemanticLogger::Loggable + + def self.create(options) + faraday = Faraday.new(options) + faraday.use Pyxis::DryRunEnforcer::FaradayBlocker + faraday.use Pyxis::Logger::FaradayLogger + + enhance_faraday(faraday) + + faraday + end + + def self.enhance_faraday(faraday) + %i[get post put patch delete].each do |method| + faraday.define_singleton_method(:"#{method}_json") do |*args, **kwargs| + response = faraday.send(method, *args, **kwargs) + json = response.body.blank? ? nil : JSON.parse(response.body) + + Thor::CoreExt::HashWithIndifferentAccess.new( + { + body: json.is_a?(Hash) ? Thor::CoreExt::HashWithIndifferentAccess.new(json) : json, + response: response, + } + ) + end + end + end + end +end diff --git a/lib/pyxis/gitlab_client.rb b/lib/pyxis/gitlab_client.rb index 6f82bc7..46543b4 100644 --- a/lib/pyxis/gitlab_client.rb +++ b/lib/pyxis/gitlab_client.rb @@ -27,30 +27,48 @@ def self.create_client(instance) 'Private-Token': File.read(client_config[:private_token]), }, } - faraday = Faraday.new(options) - faraday.use Pyxis::DryRunEnforcer::FaradayBlocker - faraday.use Pyxis::Logger::FaradayLogger - enhance_faraday(faraday) + GenericFaraday.create(options) + end + + def self.paginate_json(client, url, options = {}) + response = [] + + client.get_json(url, options).tap do |page| + response += page.body + while (next_page_link = PageLinks.new(page.response.env.response_headers).next) + page = client.get_json(next_page_link) + response += page.body + end + end - faraday + response end - def self.enhance_faraday(faraday) - %i[get post put patch delete].each do |method| - faraday.define_singleton_method(:"#{method}_json") do |*args, **kwargs| - response = faraday.send(method, *args, **kwargs) - json = response.body.blank? ? nil : JSON.parse(response.body) - - if json.is_a?(Hash) - Thor::CoreExt::HashWithIndifferentAccess.new( - { - body: Thor::CoreExt::HashWithIndifferentAccess.new(json), - response: response, - } - ) - else - response + class PageLinks + HEADER_LINK = 'link' + DELIM_LINKS = ',' + LINK_REGEX = /<([^>]+)>; rel="([^"]+)"/ + METAS = %w[last next first prev].freeze + + attr_accessor(*METAS) + + def initialize(headers) + link_header = headers[HEADER_LINK] + + extract_links(link_header) if link_header && link_header =~ /(next|first|last|prev)/ + end + + private + + def extract_links(header) + header.split(DELIM_LINKS).each do |link| + LINK_REGEX.match(link.strip) do |match| + url = match[1] + meta = match[2] + next if !url || !meta || METAS.index(meta).nil? + + send("#{meta}=", url) end end end diff --git a/lib/pyxis/managed_versioning/component_info.rb b/lib/pyxis/managed_versioning/component_info.rb index bca51d5..22c8a66 100644 --- a/lib/pyxis/managed_versioning/component_info.rb +++ b/lib/pyxis/managed_versioning/component_info.rb @@ -17,25 +17,77 @@ def execute ) return nil if pipeline.response.status == 404 - reticulum_sha = pipeline.body.sha - - components = {} - - Pyxis::Project.components.each do |project_name| - component_project_class = Pyxis::Project.const_get(project_name) - version_file = "versions/#{component_project_class.component_name}" - - begin - version_content = GithubClient.octokit.contents(Project::Reticulum.github_path, path: version_file, - ref: reticulum_sha) - version = Base64.decode64(version_content.content) - components[component_project_class.component_name] = version - rescue Octokit::NotFound - logger.warn("Version file not found for #{component_project_class.component_name} at SHA #{reticulum_sha}") - end + jobs = GitlabClient.paginate_json( + GitlabClient.client, + "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{build_id}/jobs" + ) + + manifests = find_manifests(jobs) + + components = { + reticulum: pipeline.body.sha, + } + + manifests.each do |image| + component = image.first.to_sym + next if components.key?(component) + + image_tag = image.length == 1 ? build_id : "#{build_id}-#{image.last}" + + components[component] = revision_for("code0-tech/reticulum/ci-builds/#{component}", image_tag) end - components + components.compact + end + + private + + def ghcr_client + @ghcr_client ||= GenericFaraday.create({ url: 'https://ghcr.io' }) + end + + def token_for(image) + response = ghcr_client.get_json( + 'token', + { + scope: "repository:#{image}:pull", + service: 'ghcr.io', + } + ) + + logger.warn('Failed to retrieve token', image: image) unless response.response.status == 200 + + response.body.token + end + + def revision_for(image, tag) + token = token_for(image) + + response = ghcr_client.get_json( + "v2/#{image}/manifests/#{tag}", + {}, + { + Authorization: "Bearer #{token}", + Accept: 'application/vnd.oci.image.index.v1+json', + } + ) + + logger.warn('Failed to retrieve tag for image', image: image, tag: tag) unless response.response.status == 200 + + response.body.annotations&.[]('org.opencontainers.image.version') + end + + def find_manifests(jobs) + jobs.map { |job| job['name'] } + .select { |job| job.start_with?('manifest:') } + .map { |job| job.delete_prefix('manifest:') } + .sort + .map { |job| job.split(': ') } + .map do |image| + next image if image.length == 1 + + [image.first, image.last.delete_prefix('[').delete_suffix(']')] + end end end end