From 92d5ffda82a84a559e7a0d6f8352a2071fd2d3aa Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 7 Jun 2023 09:09:01 -0500 Subject: [PATCH] Update galaxy_ng container for tests (#80721) --- changelogs/fragments/80721-ansible-galaxy.yml | 2 + lib/ansible/cli/galaxy.py | 32 ++- lib/ansible/galaxy/api.py | 9 +- .../library/reset_pulp.py | 57 ++++-- .../library/setup_collections.py | 11 +- .../tasks/download.yml | 4 +- .../ansible-galaxy-collection/tasks/init.yml | 6 + .../tasks/install.yml | 87 ++++---- .../tasks/install_offline.yml | 8 + .../ansible-galaxy-collection/tasks/main.yml | 32 ++- .../tasks/publish.yml | 5 +- .../tasks/supported_resolvelib.yml | 8 +- .../tasks/verify.yml | 8 + .../templates/ansible.cfg.j2 | 16 +- .../ansible-galaxy-collection/vars/main.yml | 3 +- .../commands/integration/cloud/galaxy.py | 188 +++++++++--------- test/units/galaxy/test_collection.py | 6 - 17 files changed, 274 insertions(+), 208 deletions(-) create mode 100644 changelogs/fragments/80721-ansible-galaxy.yml diff --git a/changelogs/fragments/80721-ansible-galaxy.yml b/changelogs/fragments/80721-ansible-galaxy.yml new file mode 100644 index 00000000000..d71d8f707bb --- /dev/null +++ b/changelogs/fragments/80721-ansible-galaxy.yml @@ -0,0 +1,2 @@ +minor_changes: +- ansible-galaxy - Remove internal configuration argument ``v3`` (https://github.com/ansible/ansible/pull/80721) diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index f67bbd46eec..70472c90929 100755 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -10,6 +10,7 @@ __metaclass__ = type # ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first from ansible.cli import CLI +import argparse import json import os.path import pathlib @@ -71,7 +72,7 @@ SERVER_DEF = [ ('password', False, 'str'), ('token', False, 'str'), ('auth_url', False, 'str'), - ('v3', False, 'bool'), + ('api_version', False, 'int'), ('validate_certs', False, 'bool'), ('client_id', False, 'str'), ('timeout', False, 'int'), @@ -79,7 +80,7 @@ SERVER_DEF = [ # config definition fields SERVER_ADDITIONAL = { - 'v3': {'default': 'False'}, + 'api_version': {'default': None, 'choices': [2, 3]}, 'validate_certs': {'cli': [{'name': 'validate_certs'}]}, 'timeout': {'default': '60', 'cli': [{'name': 'timeout'}]}, 'token': {'default': None}, @@ -240,6 +241,7 @@ class GalaxyCLI(CLI): # Common arguments that apply to more than 1 action common = opt_help.ArgumentParser(add_help=False) common.add_argument('-s', '--server', dest='api_server', help='The Galaxy API server URL') + common.add_argument('--api-version', type=int, choices=[2, 3], help=argparse.SUPPRESS) # Hidden argument that should only be used in our tests common.add_argument('--token', '--api-key', dest='api_key', help='The Ansible Galaxy API key which can be found at ' 'https://galaxy.ansible.com/me/preferences.') @@ -644,17 +646,22 @@ class GalaxyCLI(CLI): client_id = server_options.pop('client_id') token_val = server_options['token'] or NoTokenSentinel username = server_options['username'] - v3 = server_options.pop('v3') + api_version = server_options.pop('api_version') if server_options['validate_certs'] is None: server_options['validate_certs'] = context.CLIARGS['resolved_validate_certs'] validate_certs = server_options['validate_certs'] - if v3: - # This allows a user to explicitly indicate the server uses the /v3 API - # This was added for testing against pulp_ansible and I'm not sure it has - # a practical purpose outside of this use case. As such, this option is not - # documented as of now - server_options['available_api_versions'] = {'v3': '/v3'} + # This allows a user to explicitly force use of an API version when + # multiple versions are supported. This was added for testing + # against pulp_ansible and I'm not sure it has a practical purpose + # outside of this use case. As such, this option is not documented + # as of now + if api_version: + display.warning( + f'The specified "api_version" configuration for the galaxy server "{server_key}" is ' + 'not a public configuration, and may be removed at any time without warning.' + ) + server_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version} # default case if no auth info is provided. server_options['token'] = None @@ -680,6 +687,13 @@ class GalaxyCLI(CLI): )) cmd_server = context.CLIARGS['api_server'] + if context.CLIARGS['api_version']: + api_version = context.CLIARGS['api_version'] + display.warning( + 'The --api-version is not a public argument, and may be removed at any time without warning.' + ) + galaxy_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version} + cmd_token = GalaxyToken(token=context.CLIARGS['api_key']) validate_certs = context.CLIARGS['resolved_validate_certs'] diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 64b3ee0ee1d..5b049914782 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -923,10 +923,7 @@ class GalaxyAPI: data = self._call_galaxy(n_collection_url, error_context_msg=error_context_msg, cache=True) self._set_cache() - try: - signatures = data["signatures"] - except KeyError: + signatures = [signature_info["signature"] for signature_info in data.get("signatures") or []] + if not signatures: display.vvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}") - return [] - else: - return [signature_info["signature"] for signature_info in signatures] + return signatures diff --git a/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py b/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py index 53c29f7704f..c1f5e1d73ce 100644 --- a/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py +++ b/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py @@ -84,7 +84,8 @@ def invoke_api(module, url, method='GET', data=None, status_codes=None): resp, info = fetch_url(module, url, method=method, data=data, headers=headers) if info['status'] not in status_codes: - module.fail_json(url=url, **info) + info['url'] = url + module.fail_json(**info) data = to_text(resp.read()) if data: @@ -105,7 +106,7 @@ def delete_pulp_distribution(distribution, module): def delete_pulp_orphans(module): """ Deletes any orphaned pulp objects. """ - orphan_uri = module.params['pulp_api'] + '/pulp/api/v3/orphans/' + orphan_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/orphans/' task_info = invoke_api(module, orphan_uri, method='DELETE', status_codes=[202]) wait_pulp_task(task_info['task'], module) @@ -125,25 +126,39 @@ def get_galaxy_namespaces(module): return [n['name'] for n in ns_info['data']] -def get_pulp_distributions(module): +def get_pulp_distributions(module, distribution): """ Gets a list of all the pulp distributions. """ - distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/' - distro_info = invoke_api(module, distro_uri) + distro_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/distributions/ansible/ansible/' + distro_info = invoke_api(module, distro_uri + '?name=' + distribution) return [module.params['pulp_api'] + r['pulp_href'] for r in distro_info['results']] -def get_pulp_repositories(module): +def get_pulp_repositories(module, repository): """ Gets a list of all the pulp repositories. """ - repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/' - repo_info = invoke_api(module, repo_uri) + repo_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/repositories/ansible/ansible/' + repo_info = invoke_api(module, repo_uri + '?name=' + repository) return [module.params['pulp_api'] + r['pulp_href'] for r in repo_info['results']] +def get_repo_collections(repository, module): + collections_uri = module.params['galaxy_ng_server'] + 'v3/plugin/ansible/content/' + repository + '/collections/index/' + # status code 500 isn't really expected, an unhandled exception is causing this instead of a 404 + # See https://issues.redhat.com/browse/AAH-2329 + info = invoke_api(module, collections_uri + '?limit=100&offset=0', status_codes=[200, 500]) + if not info: + return [] + return [module.params['pulp_api'] + c['href'] for c in info['data']] + + +def delete_repo_collection(collection, module): + task_info = invoke_api(module, collection, method='DELETE', status_codes=[202]) + wait_pulp_task(task_info['task'], module) + + def new_galaxy_namespace(name, module): """ Creates a new namespace in Galaxy NG. """ - ns_uri = module.params['galaxy_ng_server'] + 'v3/_ui/namespaces/' - data = {'name': name, 'groups': [{'name': 'system:partner-engineers', 'object_permissions': - ['add_namespace', 'change_namespace', 'upload_to_namespace']}]} + ns_uri = module.params['galaxy_ng_server'] + 'v3/namespaces/ ' + data = {'name': name, 'groups': []} ns_info = invoke_api(module, ns_uri, method='POST', data=data, status_codes=[201]) return ns_info['id'] @@ -151,16 +166,17 @@ def new_galaxy_namespace(name, module): def new_pulp_repository(name, module): """ Creates a new pulp repository. """ - repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/' - data = {'name': name} + repo_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/repositories/ansible/ansible/' + # retain_repo_versions to work around https://issues.redhat.com/browse/AAH-2332 + data = {'name': name, 'retain_repo_versions': '1024'} repo_info = invoke_api(module, repo_uri, method='POST', data=data, status_codes=[201]) - return module.params['pulp_api'] + repo_info['pulp_href'] + return repo_info['pulp_href'] def new_pulp_distribution(name, base_path, repository, module): """ Creates a new pulp distribution for a repository. """ - distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/' + distro_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/distributions/ansible/ansible/' data = {'name': name, 'base_path': base_path, 'repository': repository} task_info = invoke_api(module, distro_uri, method='POST', data=data, status_codes=[202]) task_info = wait_pulp_task(task_info['task'], module) @@ -194,8 +210,15 @@ def main(): ) module.params['force_basic_auth'] = True - [delete_pulp_distribution(d, module) for d in get_pulp_distributions(module)] - [delete_pulp_repository(r, module) for r in get_pulp_repositories(module)] + # It may be due to the process of cleaning up orphans, but we cannot delete the namespace + # while a collection still exists, so this is just a new safety to nuke all collections + # first + for repository in module.params['repositories']: + [delete_repo_collection(c, module) for c in get_repo_collections(repository, module)] + + for repository in module.params['repositories']: + [delete_pulp_distribution(d, module) for d in get_pulp_distributions(module, repository)] + [delete_pulp_repository(r, module) for r in get_pulp_repositories(module, repository)] delete_pulp_orphans(module) [delete_galaxy_namespace(n, module) for n in get_galaxy_namespaces(module)] diff --git a/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py index 98ee1d582bf..d5083ad2fe6 100644 --- a/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py +++ b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py @@ -77,6 +77,7 @@ RETURN = ''' # ''' +import datetime import os import subprocess import tarfile @@ -104,6 +105,7 @@ def publish_collection(module, collection): collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (namespace, name, version)) b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict') os.mkdir(b_collection_dir) + os.mkdir(os.path.join(b_collection_dir, b'meta')) with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd: fd.write(b"Collection readme") @@ -120,6 +122,8 @@ def publish_collection(module, collection): } with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd: fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict')) + with open(os.path.join(b_collection_dir, b'meta/runtime.yml'), mode='wb') as fd: + fd.write(b'requires_ansible: ">=1.0.0"') with tempfile.NamedTemporaryFile(mode='wb') as temp_fd: temp_fd.write(b"data") @@ -241,7 +245,8 @@ def run_module(): supports_check_mode=False ) - result = dict(changed=True, results=[]) + start = datetime.datetime.now() + result = dict(changed=True, results=[], start=str(start)) pool = threading.Pool(4) publish_func = partial(publish_collection, module) @@ -258,7 +263,9 @@ def run_module(): r['build']['rc'] + r['publish']['rc'] for r in result['results'] )) - module.exit_json(failed=failed, **result) + end = datetime.datetime.now() + delta = end - start + module.exit_json(failed=failed, end=str(end), delta=str(delta), **result) def main(): diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml index b651a73e73e..a554c277b2a 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml @@ -5,7 +5,7 @@ state: directory - name: download collection with multiple dependencies with --no-deps - command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 --no-deps -s pulp_v2 {{ galaxy_verbosity }} + command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 --no-deps -s galaxy_ng {{ galaxy_verbosity }} register: download_collection args: chdir: '{{ galaxy_dir }}/download' @@ -34,7 +34,7 @@ - (download_collection_actual.files[1].path | basename) in ['requirements.yml', 'parent_dep-parent_collection-1.0.0.tar.gz'] - name: download collection with multiple dependencies - command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 -s pulp_v2 {{ galaxy_verbosity }} + command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 -s galaxy_ng {{ galaxy_verbosity }} register: download_collection args: chdir: '{{ galaxy_dir }}/download' diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/init.yml b/test/integration/targets/ansible-galaxy-collection/tasks/init.yml index c4b5d9c5102..46198fef390 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/init.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/init.yml @@ -5,6 +5,12 @@ chdir: '{{ galaxy_dir }}/scratch' register: init_relative +- name: create required runtime.yml + copy: + content: | + requires_ansible: '>=1.0.0' + dest: '{{ galaxy_dir }}/scratch/ansible_test/my_collection/meta/runtime.yml' + - name: get result of create default skeleton find: path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection' diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml index b332308ca2f..39bd6f3d780 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml @@ -168,7 +168,7 @@ - name: Find artifact url for namespace3.name uri: - url: '{{ test_server }}{{ vX }}collections/namespace3/name/versions/1.0.0/' + url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/namespace3/name/versions/1.0.0/' user: '{{ pulp_user }}' password: '{{ pulp_password }}' force_basic_auth: true @@ -218,7 +218,7 @@ state: absent - assert: - that: error == expected_error + that: expected_error in error vars: error: "{{ result.stderr | regex_replace('\\n', ' ') }}" expected_error: >- @@ -258,12 +258,14 @@ ignore_errors: yes register: result - - debug: msg="Actual - {{ error }}" + - debug: + msg: "Actual - {{ error }}" - - debug: msg="Expected - {{ expected_error }}" + - debug: + msg: "Expected - {{ expected_error }}" - assert: - that: error == expected_error + that: expected_error in error always: - name: clean up collection skeleton and artifact file: @@ -295,7 +297,7 @@ - name: Find artifact url for namespace4.name uri: - url: '{{ test_server }}{{ vX }}collections/namespace4/name/versions/1.0.0/' + url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/namespace4/name/versions/1.0.0/' user: '{{ pulp_user }}' password: '{{ pulp_password }}' force_basic_auth: true @@ -325,10 +327,11 @@ environment: ANSIBLE_GALAXY_SERVER_LIST: undefined -- when: not requires_auth +# pulp_v2 doesn't require auth +- when: v2|default(false) block: - name: install a collection with an empty server list - {{ test_id }} - command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' {{ galaxy_verbosity }} + command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' --api-version 2 {{ galaxy_verbosity }} register: install_empty_server_list environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' @@ -571,7 +574,6 @@ - namespace8 - namespace9 -# SIVEL - name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_id }} assert: that: @@ -697,39 +699,40 @@ - namespace8 - namespace9 -# Uncomment once pulp container is at pulp>=0.5.0 -#- name: install cache.cache at the current latest version -# command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' -vvv -# environment: -# ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -# -#- set_fact: -# cache_version_build: '{{ (cache_version_build | int) + 1 }}' -# -#- name: publish update for cache.cache test -# setup_collections: -# server: galaxy_ng -# collections: -# - namespace: cache -# name: cache -# version: 1.0.{{ cache_version_build }} -# -#- name: make sure the cache version list is ignored on a collection version change - {{ test_id }} -# command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv -# register: install_cached_update -# environment: -# ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -# -#- name: get result of cache version list is ignored on a collection version change - {{ test_id }} -# slurp: -# path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json' -# register: install_cached_update_actual -# -#- name: assert cache version list is ignored on a collection version change - {{ test_id }} -# assert: -# that: -# - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout' -# - (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build +- when: not v2|default(false) + block: + - name: install cache.cache at the current latest version + command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' -vvv + environment: + ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' + + - set_fact: + cache_version_build: '{{ (cache_version_build | int) + 1 }}' + + - name: publish update for cache.cache test + setup_collections: + server: galaxy_ng + collections: + - namespace: cache + name: cache + version: 1.0.{{ cache_version_build }} + + - name: make sure the cache version list is ignored on a collection version change - {{ test_id }} + command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv + register: install_cached_update + environment: + ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' + + - name: get result of cache version list is ignored on a collection version change - {{ test_id }} + slurp: + path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json' + register: install_cached_update_actual + + - name: assert cache version list is ignored on a collection version change - {{ test_id }} + assert: + that: + - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout' + - (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build - name: install collection with symlink - {{ test_id }} command: ansible-galaxy collection install symlink.symlink -s '{{ test_name }}' {{ galaxy_verbosity }} diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install_offline.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install_offline.yml index 74c998382b2..f3b9777c81d 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/install_offline.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/install_offline.yml @@ -25,6 +25,14 @@ regexp: "^dependencies:*" line: "dependencies: {'ns.coll2': '>=1.0.0'}" + - name: create required runtime.yml + copy: + dest: "{{ galaxy_dir }}/offline/setup/ns/{{ item }}/meta/runtime.yml" + content: "requires_ansible: '>=1.0.0'" + loop: + - coll1 + - coll2 + - name: build both collections command: ansible-galaxy collection build {{ init_dir }}/ns/{{ item }} args: diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml index 724c861e696..3a9d9e6fab6 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml @@ -72,13 +72,12 @@ vars: test_name: '{{ item.name }}' test_server: '{{ item.server }}' - vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}' + test_api_server: '{{ item.api_server|default(item.server) }}' loop: - name: pulp_v2 - server: '{{ pulp_server }}published/api/' - - name: pulp_v3 - server: '{{ pulp_server }}published/api/' - v3: true + api_server: '{{ galaxy_ng_server }}' + server: '{{ pulp_server }}primary/api/' + v2: true - name: galaxy_ng server: '{{ galaxy_ng_server }}' v3: true @@ -108,8 +107,9 @@ test_id: '{{ item.name }}' test_name: '{{ item.name }}' test_server: '{{ item.server }}' - vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}' + test_api_server: '{{ item.api_server|default(item.server) }}' requires_auth: '{{ item.requires_auth|default(false) }}' + v2: '{{ item.v2|default(false) }}' args: apply: environment: @@ -120,10 +120,9 @@ v3: true requires_auth: true - name: pulp_v2 - server: '{{ pulp_server }}published/api/' - - name: pulp_v3 - server: '{{ pulp_server }}published/api/' - v3: true + server: '{{ pulp_server }}primary/api/' + api_server: '{{ galaxy_ng_server }}' + v2: true - name: test installing and downloading collections with the range of supported resolvelib versions include_tasks: supported_resolvelib.yml @@ -176,13 +175,13 @@ in install_cross_dep.stdout # pulp_v2 is highest in the list so it will find it there first - >- - "'parent_dep.parent_collection:1.0.0' obtained from server pulp_v2" + "'parent_dep.parent_collection:1.0.0' obtained from server galaxy_ng" in install_cross_dep.stdout - >- - "'child_dep.child_collection:0.9.9' obtained from server pulp_v2" + "'child_dep.child_collection:0.9.9' obtained from server galaxy_ng" in install_cross_dep.stdout - >- - "'child_dep.child_dep2:1.2.2' obtained from server pulp_v2" + "'child_dep.child_dep2:1.2.2' obtained from server galaxy_ng" in install_cross_dep.stdout - (install_cross_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' - (install_cross_dep_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' @@ -204,10 +203,9 @@ ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}' ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' vars: - test_api_fallback: 'pulp_v2' - test_api_fallback_versions: 'v1, v2' - test_name: 'galaxy_ng' - test_server: '{{ galaxy_ng_server }}' + test_api_fallback: 'galaxy_ng' + test_api_fallback_versions: 'v3, pulp-v3, v1' + test_name: 'pulp_v2' - name: run ansible-galaxy collection list tests include_tasks: list.yml diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/publish.yml b/test/integration/targets/ansible-galaxy-collection/tasks/publish.yml index 241eae602d0..1be16ae9015 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/publish.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/publish.yml @@ -5,9 +5,12 @@ chdir: '{{ galaxy_dir }}' register: publish_collection +- name: ensure we can download the published collection - {{ test_name }} + command: ansible-galaxy collection install -s {{ test_name }} -p "{{ remote_tmp_dir }}/publish/{{ test_name }}" ansible_test.my_collection==1.0.0 {{ galaxy_verbosity }} + - name: get result of publish collection - {{ test_name }} uri: - url: '{{ test_server }}{{ vX }}collections/ansible_test/my_collection/versions/1.0.0/' + url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/ansible_test/my_collection/versions/1.0.0/' return_content: yes user: '{{ pulp_user }}' password: '{{ pulp_password }}' diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml b/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml index 763c5a19f7f..bff3689275a 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml @@ -20,11 +20,11 @@ - include_tasks: install.yml vars: - test_name: pulp_v3 + test_name: galaxy_ng test_id: '{{ test_name }} (resolvelib {{ resolvelib_version }})' - test_server: '{{ pulp_server }}published/api/' - vX: "v3/" - requires_auth: false + test_server: '{{ galaxy_ng_server }}' + test_api_server: '{{ galaxy_ng_server }}' + requires_auth: true args: apply: environment: diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml b/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml index dfe3d0f7142..0fe2f82d16f 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml @@ -3,6 +3,11 @@ args: chdir: '{{ galaxy_dir }}/scratch' +- name: created required runtime.yml + copy: + content: 'requires_ansible: ">=1.0.0"' + dest: '{{ galaxy_dir }}/scratch/ansible_test/verify/meta/runtime.yml' + - name: build the collection command: ansible-galaxy collection build scratch/ansible_test/verify args: @@ -31,6 +36,9 @@ - name: verify the collection against the first valid server command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -vvvv {{ galaxy_verbosity }} register: verify + vars: + # This sets a specific precedence that the tests are expecting + ANSIBLE_GALAXY_SERVER_LIST: offline,secondary,pulp_v2,galaxy_ng - assert: that: diff --git a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 index 9bff527bb26..a242979d227 100644 --- a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 +++ b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 @@ -1,28 +1,22 @@ [galaxy] # Ensures subsequent unstable reruns don't use the cached information causing another failure cache_dir={{ remote_tmp_dir }}/galaxy_cache -server_list=offline,pulp_v2,pulp_v3,galaxy_ng,secondary +server_list=offline,galaxy_ng,secondary,pulp_v2 [galaxy_server.offline] url={{ offline_server }} [galaxy_server.pulp_v2] -url={{ pulp_server }}published/api/ -username={{ pulp_user }} -password={{ pulp_password }} - -[galaxy_server.pulp_v3] -url={{ pulp_server }}published/api/ -v3=true +url={{ pulp_server }}primary/api/ username={{ pulp_user }} password={{ pulp_password }} +api_version=2 [galaxy_server.galaxy_ng] -url={{ galaxy_ng_server }} +url={{ galaxy_ng_server }}content/primary/ token={{ galaxy_ng_token.json.token }} [galaxy_server.secondary] -url={{ pulp_server }}secondary/api/ -v3=true +url={{ galaxy_ng_server }}content/secondary/ username={{ pulp_user }} password={{ pulp_password }} diff --git a/test/integration/targets/ansible-galaxy-collection/vars/main.yml b/test/integration/targets/ansible-galaxy-collection/vars/main.yml index d3f188b9061..d3d9beea8e4 100644 --- a/test/integration/targets/ansible-galaxy-collection/vars/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/vars/main.yml @@ -17,11 +17,12 @@ unsupported_resolvelib_versions: - "0.5.1" pulp_repositories: - - published + - primary - secondary publish_namespaces: - ansible_test + - secondary collection_list: # Scenario to test out pre-release being ignored unless explicitly set and version pagination. diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/galaxy.py b/test/lib/ansible_test/_internal/commands/integration/cloud/galaxy.py index ebe8d1ca11a..5ea10cbaeb4 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/galaxy.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/galaxy.py @@ -10,12 +10,21 @@ from ....config import ( from ....docker_util import ( docker_cp_to, + docker_exec, ) from ....containers import ( run_support_container, ) +from ....encoding import ( + to_text, +) + +from ....util import ( + display, +) + from . import ( CloudEnvironment, CloudEnvironmentConfig, @@ -23,53 +32,59 @@ from . import ( ) -# We add BasicAuthentication, to make the tasks that deal with -# direct API access easier to deal with across galaxy_ng and pulp -SETTINGS = ''' -CONTENT_ORIGIN = 'http://{host}:80' -ANSIBLE_API_HOSTNAME = 'http://{host}:80' -ANSIBLE_CONTENT_HOSTNAME = 'http://{host}:80/pulp/content' -TOKEN_AUTH_DISABLED = True -GALAXY_REQUIRE_CONTENT_APPROVAL = False -GALAXY_AUTHENTICATION_CLASSES = [ - "rest_framework.authentication.SessionAuthentication", - "rest_framework.authentication.TokenAuthentication", - "rest_framework.authentication.BasicAuthentication", -] -''' - -SET_ADMIN_PASSWORD = b'''#!/usr/bin/execlineb -S0 -foreground { - redirfd -w 1 /dev/null - redirfd -w 2 /dev/null - export DJANGO_SETTINGS_MODULE pulpcore.app.settings - export PULP_CONTENT_ORIGIN localhost - s6-setuidgid postgres - if { /usr/local/bin/django-admin reset-admin-password --password password } - if { /usr/local/bin/pulpcore-manager create-group system:partner-engineers --users admin } -} -''' - -# There are 2 overrides here: -# 1. Change the gunicorn bind address from 127.0.0.1 to 0.0.0.0 now that Galaxy NG does not allow us to access the -# Pulp API through it. -# 2. Grant access allowing us to DELETE a namespace in Galaxy NG. This is as CI deletes and recreates repos and -# distributions in Pulp which now breaks the namespace in Galaxy NG. Recreating it is the "simple" fix to get it -# working again. -# These may not be needed in the future, especially if 1 becomes configurable by an env var but for now they must be -# done. -OVERRIDES = b'''#!/usr/bin/execlineb -S0 -foreground { - sed -i "0,/\\"127.0.0.1:24817\\"/s//\\"0.0.0.0:24817\\"/" /etc/services.d/pulpcore-api/run +GALAXY_HOST_NAME = 'galaxy-pulp' +SETTINGS = { + 'PULP_CONTENT_ORIGIN': f'http://{GALAXY_HOST_NAME}', + 'PULP_ANSIBLE_API_HOSTNAME': f'http://{GALAXY_HOST_NAME}', + 'PULP_GALAXY_API_PATH_PREFIX': '/api/galaxy/', + # These paths are unique to the container image which has an nginx location for /pulp/content to route + # requests to the content backend + 'PULP_ANSIBLE_CONTENT_HOSTNAME': f'http://{GALAXY_HOST_NAME}/pulp/content/api/galaxy/v3/artifacts/collections/', + 'PULP_CONTENT_PATH_PREFIX': '/pulp/content/api/galaxy/v3/artifacts/collections/', + 'PULP_GALAXY_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.TokenAuthentication', + 'rest_framework.authentication.BasicAuthentication', + 'django.contrib.auth.backends.ModelBackend', + ], + # This should probably be false see https://issues.redhat.com/browse/AAH-2328 + 'PULP_GALAXY_REQUIRE_CONTENT_APPROVAL': 'true', + 'PULP_GALAXY_DEPLOYMENT_MODE': 'standalone', + 'PULP_GALAXY_AUTO_SIGN_COLLECTIONS': 'false', + 'PULP_GALAXY_COLLECTION_SIGNING_SERVICE': 'ansible-default', + 'PULP_RH_ENTITLEMENT_REQUIRED': 'insights', + 'PULP_TOKEN_AUTH_DISABLED': 'false', + 'PULP_TOKEN_SERVER': f'http://{GALAXY_HOST_NAME}/token/', + 'PULP_TOKEN_SIGNATURE_ALGORITHM': 'ES256', + 'PULP_PUBLIC_KEY_PATH': '/src/galaxy_ng/dev/common/container_auth_public_key.pem', + 'PULP_PRIVATE_KEY_PATH': '/src/galaxy_ng/dev/common/container_auth_private_key.pem', + 'PULP_ANALYTICS': 'false', + 'PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_ACCESS': 'true', + 'PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD': 'true', + 'PULP_GALAXY_ENABLE_LEGACY_ROLES': 'true', + 'PULP_GALAXY_FEATURE_FLAGS__execution_environments': 'false', + 'PULP_SOCIAL_AUTH_LOGIN_REDIRECT_URL': '/', + 'PULP_GALAXY_FEATURE_FLAGS__ai_deny_index': 'true', + 'PULP_DEFAULT_ADMIN_PASSWORD': 'password' } -# This sed calls changes the first occurrence to "allow" which is conveniently the delete operation for a namespace. -# https://github.com/ansible/galaxy_ng/blob/master/galaxy_ng/app/access_control/statements/standalone.py#L9-L11. -backtick NG_PREFIX { python -c "import galaxy_ng; print(galaxy_ng.__path__[0], end='')" } -importas ng_prefix NG_PREFIX -foreground { - sed -i "0,/\\"effect\\": \\"deny\\"/s//\\"effect\\": \\"allow\\"/" ${ng_prefix}/app/access_control/statements/standalone.py -}''' + +GALAXY_IMPORTER = b''' +[galaxy-importer] +ansible_local_tmp=~/.ansible/tmp +ansible_test_local_image=false +check_required_tags=false +check_runtime_yaml=false +check_changelog=false +infra_osd=false +local_image_docker=false +log_level_main=INFO +require_v1_or_greater=false +run_ansible_doc=false +run_ansible_lint=false +run_ansible_test=false +run_flake8=false +'''.strip() class GalaxyProvider(CloudProvider): @@ -81,13 +96,9 @@ class GalaxyProvider(CloudProvider): def __init__(self, args: IntegrationConfig) -> None: super().__init__(args) - # Cannot use the latest container image as either galaxy_ng 4.2.0rc2 or pulp 0.5.0 has sporatic issues with - # dropping published collections in CI. Try running the tests multiple times when updating. Will also need to - # comment out the cache tests in 'test/integration/targets/ansible-galaxy-collection/tasks/install.yml' when - # the newer update is available. - self.pulp = os.environ.get( + self.image = os.environ.get( 'ANSIBLE_PULP_CONTAINER', - 'quay.io/ansible/pulp-galaxy-ng:b79a7be64eff' + 'quay.io/pulp/galaxy:4.7.1' ) self.uses_docker = True @@ -96,46 +107,45 @@ class GalaxyProvider(CloudProvider): """Setup cloud resource before delegation and reg cleanup callback.""" super().setup() - galaxy_port = 80 - pulp_port = 24817 - - ports = [ - galaxy_port, - pulp_port, - ] - - # Create the container, don't run it, we need to inject configs before it starts - descriptor = run_support_container( - self.args, - self.platform, - self.pulp, - 'galaxy-pulp', - ports, - start=False, - ) + with tempfile.NamedTemporaryFile(mode='w+') as env_fd: + settings = '\n'.join( + f'{key}={value}' for key, value in SETTINGS.items() + ) + env_fd.write(settings) + env_fd.flush() + display.info(f'>>> galaxy_ng Configuration\n{settings}', verbosity=3) + descriptor = run_support_container( + self.args, + self.platform, + self.image, + GALAXY_HOST_NAME, + [ + 80, + ], + aliases=[ + GALAXY_HOST_NAME, + ], + start=True, + options=[ + '--env-file', env_fd.name, + ], + ) if not descriptor: return - pulp_id = descriptor.container_id - - injected_files = { - '/etc/pulp/settings.py': SETTINGS.format(host=descriptor.name).encode(), - '/etc/cont-init.d/111-postgres': SET_ADMIN_PASSWORD, - '/etc/cont-init.d/000-ansible-test-overrides': OVERRIDES, - } - - for path, content in injected_files.items(): + injected_files = [ + ('/etc/galaxy-importer/galaxy-importer.cfg', GALAXY_IMPORTER, 'galaxy-importer'), + ] + for path, content, friendly_name in injected_files: with tempfile.NamedTemporaryFile() as temp_fd: temp_fd.write(content) temp_fd.flush() - docker_cp_to(self.args, pulp_id, temp_fd.name, path) - - descriptor.start(self.args) + display.info(f'>>> {friendly_name} Configuration\n{to_text(content)}', verbosity=3) + docker_exec(self.args, descriptor.container_id, ['mkdir', '-p', os.path.dirname(path)], True) + docker_cp_to(self.args, descriptor.container_id, temp_fd.name, path) - self._set_cloud_config('PULP_HOST', descriptor.name) - self._set_cloud_config('PULP_PORT', str(pulp_port)) - self._set_cloud_config('GALAXY_PORT', str(galaxy_port)) + self._set_cloud_config('PULP_HOST', GALAXY_HOST_NAME) self._set_cloud_config('PULP_USER', 'admin') self._set_cloud_config('PULP_PASSWORD', 'password') @@ -148,21 +158,19 @@ class GalaxyEnvironment(CloudEnvironment): pulp_user = str(self._get_cloud_config('PULP_USER')) pulp_password = str(self._get_cloud_config('PULP_PASSWORD')) pulp_host = self._get_cloud_config('PULP_HOST') - galaxy_port = self._get_cloud_config('GALAXY_PORT') - pulp_port = self._get_cloud_config('PULP_PORT') return CloudEnvironmentConfig( ansible_vars=dict( pulp_user=pulp_user, pulp_password=pulp_password, - pulp_api='http://%s:%s' % (pulp_host, pulp_port), - pulp_server='http://%s:%s/pulp_ansible/galaxy/' % (pulp_host, pulp_port), - galaxy_ng_server='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port), + pulp_api=f'http://{pulp_host}', + pulp_server=f'http://{pulp_host}/pulp_ansible/galaxy/', + galaxy_ng_server=f'http://{pulp_host}/api/galaxy/', ), env_vars=dict( PULP_USER=pulp_user, PULP_PASSWORD=pulp_password, - PULP_SERVER='http://%s:%s/pulp_ansible/galaxy/api/' % (pulp_host, pulp_port), - GALAXY_NG_SERVER='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port), + PULP_SERVER=f'http://{pulp_host}/pulp_ansible/galaxy/api/', + GALAXY_NG_SERVER=f'http://{pulp_host}/api/galaxy/', ), ) diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py index 6e8d50000ab..54b54ada56c 100644 --- a/test/units/galaxy/test_collection.py +++ b/test/units/galaxy/test_collection.py @@ -223,23 +223,19 @@ def test_cli_options(required_signature_count, valid, monkeypatch): { 'url': 'https://galaxy.ansible.com', 'validate_certs': 'False', - 'v3': 'False', }, # Expected server attributes { 'validate_certs': False, - '_available_api_versions': {}, }, ), ( { 'url': 'https://galaxy.ansible.com', 'validate_certs': 'True', - 'v3': 'True', }, { 'validate_certs': True, - '_available_api_versions': {'v3': '/v3'}, }, ), ], @@ -257,7 +253,6 @@ def test_bool_type_server_config_options(config, server, monkeypatch): "server_list=server1\n", "[galaxy_server.server1]", "url=%s" % config['url'], - "v3=%s" % config['v3'], "validate_certs=%s\n" % config['validate_certs'], ] @@ -277,7 +272,6 @@ def test_bool_type_server_config_options(config, server, monkeypatch): assert galaxy_cli.api_servers[0].name == 'server1' assert galaxy_cli.api_servers[0].validate_certs == server['validate_certs'] - assert galaxy_cli.api_servers[0]._available_api_versions == server['_available_api_versions'] @pytest.mark.parametrize('global_ignore_certs', [True, False])