apt: recreate apt cache if /var/lib/apt/lists is missing (#86612)

* While creating bare minimum container images, sometimes
  /var/lib/apt/lists is removed.
  Recreate this directory in order to update cache successfully.

Fixes: #61176

Signed-off-by: Abhijeet Kasurde <Akasurde@redhat.com>
Co-authored-by: Sloane Hertel <19572925+shertel@users.noreply.github.com>

changelogs/fragments/apt_update.yml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+    - apt - recreate the APT lists directory (/var/lib/apt/lists by default) if missing (https://github.com/ansible/ansible/issues/61176).

lib/ansible/modules/apt.py

@@ -224,6 +224,7 @@ notes:
    - If the interpreter can't import C(python3-apt) the module will check for it in system-owned interpreters as well.
      If the dependency can't be found, depending on the value of O(auto_install_module_deps) the module will attempt to install it.
      If the dependency is found or installed, the module will be respawned under the correct interpreter.
+   - From ansible-core 2.21, if the apt lists directory is absent, the module will recreate it by running C(apt-get update).
 """
 
 EXAMPLES = """
@@ -1224,6 +1225,17 @@ def get_updated_cache_time():
     return mtimestamp, updated_cache_time
 
 
+def recreate_cache(module, max_retries=2):
+    retries = 0
+    update_cmd = ['apt-get', 'update', '-q']
+    while retries < max_retries:
+        rc, stdout, stderr = module.run_command(update_cmd)
+        retries += 1
+        if rc == 0:
+            break
+    return rc, stdout, stderr
+
+
 # https://github.com/ansible/ansible-modules-core/issues/2951
 def get_cache(module):
     """Attempt to get the cache object and update till it works"""
@@ -1233,18 +1245,20 @@ def get_cache(module):
     except SystemError as e:
         if '/var/lib/apt/lists/' in to_native(e).lower():
             # update cache until files are fixed or retries exceeded
-            retries = 0
-            while retries < 2:
-                (rc, so, se) = module.run_command(['apt-get', 'update', '-q'])
-                retries += 1
-                if rc == 0:
-                    break
+            rc, stdout, stderr = recreate_cache(module)
             if rc != 0:
-                module.fail_json(msg='Updating the cache to correct corrupt package lists failed:\n%s\n%s' % (to_native(e), so + se), rc=rc)
+                module.fail_json(msg=f'Updating the cache to correct corrupt package lists failed:\n{to_native(e)}\n{stdout + stderr}', rc=rc)
             # try again
             cache = apt.Cache()
         else:
             module.fail_json(msg=to_native(e))
+
+    # Check if the cache is valid
+    if not os.path.isdir(apt_pkg.config.find_dir("Dir::State::Lists")):
+        rc, stdout, stderr = recreate_cache(module)
+        if rc != 0:
+            module.fail_json(msg=f'Failed to recreate the cache: {stdout + stderr}', rc=rc)
+        cache = apt.Cache()
     return cache
 
 

test/integration/targets/apt/tasks/apt_update.yml

@@ -0,0 +1,24 @@
+- block:
+    - name: Create a backup of the /var/lib/apt/lists/partial
+      command: mv /var/lib/apt/lists/partial /var/lib/apt/lists/partial.backup
+
+    - name: Verify that apt update does not fail if the /var/lib/apt/lists/partial is not present
+      apt:
+        update_cache: yes
+
+    - name: Create a backup of the /var/lib/apt/lists
+      command: mv /var/lib/apt/lists /var/lib/apt/lists.backup
+
+    - name: Verify that apt update does not fail if the /var/lib/apt/lists is not present
+      apt:
+        update_cache: yes
+
+  always:
+    - name: Check if the /var/lib/apt/lists is present
+      stat:
+        path: /var/lib/apt/lists.backup
+      register: apt_lists_backup_stat
+
+    - name: Restore the /var/lib/apt/lists
+      shell: rsync -a --delete /var/lib/apt/lists.backup/ /var/lib/apt/lists/ && rm -rf /var/lib/apt/lists.backup
+      when: apt_lists_backup_stat.stat.exists

test/integration/targets/apt/tasks/main.yml

@@ -16,6 +16,8 @@
 
   - import_tasks: 'apt-builddep.yml'
 
+  - import_tasks: 'apt_update.yml'
+
   - block:
       - import_tasks: 'repo.yml'
     always: