add retry logic to flaky commands

README.md

@@ -72,6 +72,19 @@ you are not running a stable release.
     - [OpenShift Enterprise](https://docs.openshift.com/enterprise/latest/install_config/install/advanced_install.html)
     - [OpenShift Origin](https://docs.openshift.org/latest/install_config/install/advanced_install.html)
 
+## Running
+
+Running playbooks directly from openshift-ansible requires changing
+the current directory to the root of the content so that the provided
+`ansible.cfg`, plugins, and roles are used.  If running from a clone of
+the repository then this means using `cd` to change to the repository
+root after checking out. If running from the RPM install then this means
+doing `cd /usr/share/ansible/openshift-ansible/` before running playbooks.
+
+Users of containerized openshift-ansible should consult
+[README_CONTAINERIZED_INSTALLATION.md](README_CONTAINERIZED_INSTALLATION.md)
+for usage directions.
+
 ## Containerized OpenShift Ansible
 
 See [README_CONTAINER_IMAGE.md](README_CONTAINER_IMAGE.md) for information on how to package openshift-ansible as a container image.

action_plugins/openshift_package.py

@@ -0,0 +1,98 @@
+# (c) 2017, Red Hat Inc.
+#
+# This is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+This is a copy of the "package" action plugin from Ansible,
+with minor modifications to implement retries.
+"""
+
+# pylint: skip-file
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import time
+
+from ansible.plugins.action import ActionBase
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class ActionModule(ActionBase):
+
+    TRANSFERS_FILES = False
+
+    def run(self, tmp=None, task_vars=None):
+        ''' handler for package operations '''
+
+        self._supports_check_mode = True
+        self._supports_async = True
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+
+        module = self._task.args.get('use', 'auto')
+
+        if module == 'auto':
+            try:
+                if self._task.delegate_to:  # if we delegate, we should use delegated host's facts
+                    module = self._templar.template("{{hostvars['%s']['ansible_pkg_mgr']}}" % self._task.delegate_to)
+                else:
+                    module = self._templar.template('{{ansible_pkg_mgr}}')
+            except Exception:
+                pass  # could not get it from template!
+
+        if module == 'auto':
+            module_args = dict(filter='ansible_pkg_mgr', gather_subset='!all')
+            facts = self._execute_module(module_name='setup', module_args=module_args, task_vars=task_vars)
+            display.debug("Facts %s" % facts)
+            if 'ansible_facts' in facts and 'ansible_pkg_mgr' in facts['ansible_facts']:
+                module = getattr(facts['ansible_facts'], 'ansible_pkg_mgr', 'auto')
+
+        if module != 'auto':
+
+            if module not in self._shared_loader_obj.module_loader:
+                result['failed'] = True
+                result['msg'] = 'Could not find a module for %s.' % module
+            else:
+                # run the 'package' module
+                new_module_args = self._task.args.copy()
+                if 'use' in new_module_args:
+                    del new_module_args['use']
+
+                display.vvvv("Running {}".format(module))
+                tries = 0
+                while True:
+                    res = self._execute_module(
+                        module_name=module,
+                        module_args=new_module_args,
+                        task_vars=task_vars,
+                        wrap_async=self._task.async,
+                    )
+                    tries += 1
+                    if tries > 3 or not res.get('failed'):
+                        result.update(res)
+                        break
+                    result['last_failed'] = res
+                    display.v("{} module failed on try {} with result: {}".format(module, tries, res))
+                    time.sleep(5)
+        else:
+            result['failed'] = True
+            result['msg'] = 'Could not detect which package manager to use. Try gathering facts or setting the "use" option.'
+
+        return result

ansible.cfg

@@ -11,6 +11,8 @@
 
 # Additional default options for OpenShift Ansible
 callback_plugins = callback_plugins/
+action_plugins = action_plugins/
+library = library/
 forks = 20
 host_key_checking = False
 retry_files_enabled = False

library/openshift_package.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2017, Red Hat, Inc.
+#
+# This is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+This is a copy of the "package" module from Ansible with the name changed.
+"""
+
+ANSIBLE_METADATA = {'metadata_version': '1.0',
+                    'status': ['stableinterface'],
+                    'supported_by': 'community'}
+
+
+DOCUMENTATION = '''
+---
+module: openshift_package
+version_added: 2.0
+author:
+    - Red Hat Inc
+short_description: Generic OS package manager
+description:
+     - Installs, upgrade and removes packages using the underlying OS package manager, with retries on failure.
+options:
+  name:
+    description:
+      - "Package name, or package specifier with version, like C(name-1.0)."
+      - "Be aware that packages are not always named the same and this module will not 'translate' them per distro."
+    required: true
+  state:
+    description:
+      - Whether to install (C(present), C(latest)), or remove (C(absent)) a package.
+    required: true
+  use:
+    description:
+      - The required package manager module to use (yum, apt, etc). The default 'auto' will use existing facts or try to autodetect it.
+      - You should only use this field if the automatic selection is not working for some reason.
+    required: false
+    default: auto
+requirements:
+    - Whatever is required for the package plugins specific for each system.
+notes:
+    - This module actually calls the pertinent package modules for each system (apt, yum, etc).
+'''
+EXAMPLES = '''
+- name: install the latest version of ntpdate
+  openshift_package:
+    name: ntpdate
+    state: latest
+'''

lookup_plugins/oo_option.py

@@ -25,7 +25,7 @@
 except ImportError:
     # ansible-1.9.x
     class LookupBase(object):
-        def __init__(self, basedir=None, runner=None, **kwargs):
+        def __init__(self, basedir=None, runner=None, **_):
             self.runner = runner
             self.basedir = self.runner.basedir
 
@@ -40,23 +40,11 @@ def get_basedir(self, variables):
 class LookupModule(LookupBase):
     ''' oo_option lookup plugin main class '''
 
-    # Reason: disable unused-argument because Ansible is calling us with many
-    #     parameters we are not interested in.
-    #     The lookup plugins of Ansible have this kwargs “catch-all” parameter
-    #     which is not used
-    # Status: permanently disabled unless Ansible API evolves
-    # pylint: disable=unused-argument
-    def __init__(self, basedir=None, **kwargs):
+    def __init__(self, basedir=None, **_):
         ''' Constructor '''
         self.basedir = basedir
 
-    # Reason: disable unused-argument because Ansible is calling us with many
-    #     parameters we are not interested in.
-    #     The lookup plugins of Ansible have this kwargs “catch-all” parameter
-    #     which is not used
-    # Status: permanently disabled unless Ansible API evolves
-    # pylint: disable=unused-argument
-    def run(self, terms, variables, **kwargs):
+    def run(self, terms, variables, **_):
         ''' Main execution path '''
 
         ret = []

playbooks/action_plugins

@@ -0,0 +1 @@
+../action_plugins

playbooks/adhoc/action_plugins

@@ -0,0 +1 @@
+../../action_plugins

playbooks/adhoc/contiv/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/adhoc/create_pv/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/adhoc/docker_loopback_to_lvm/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/adhoc/docker_storage_cleanup/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/adhoc/grow_docker_vg/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/adhoc/grow_docker_vg/filter_plugins/action_plugins

@@ -0,0 +1 @@
+../../../../action_plugins

playbooks/adhoc/sdn_restart/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/adhoc/uninstall.yml

@@ -102,7 +102,7 @@
   - block:
     - block:
       - name: Remove packages
-        package: name={{ item }} state=absent
+        openshift_package: name={{ item }} state=absent
         with_items:
         - atomic-enterprise
         - atomic-enterprise-node
@@ -135,7 +135,7 @@
         - tuned-profiles-origin-node
 
       - name: Remove flannel package
-        package: name=flannel state=absent
+        openshift_package: name=flannel state=absent
         when: openshift_use_flannel | default(false) | bool
       when: not is_atomic | bool
 
@@ -344,7 +344,7 @@
     - atomic-openshift-master
 
   - name: Remove packages
-    package: name={{ item }} state=absent
+    openshift_package: name={{ item }} state=absent
     when: not is_atomic | bool and openshift_remove_all | default(True) | bool
     with_items:
     - atomic-enterprise
@@ -485,7 +485,7 @@
     failed_when: false
 
   - name: Remove packages
-    package: name={{ item }} state=absent
+    openshift_package: name={{ item }} state=absent
     when: not is_atomic | bool and openshift_remove_all | default(True) | bool
     with_items:
     - etcd
@@ -543,7 +543,7 @@
     - firewalld
 
   - name: Remove packages
-    package: name={{ item }} state=absent
+    openshift_package: name={{ item }} state=absent
     when: not is_atomic | bool and openshift_remove_all | default(True) | bool
     with_items:
     - haproxy

playbooks/aws/action_plugins

@@ -0,0 +1 @@
+../../action_plugins

playbooks/aws/openshift-cluster/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/action_plugins

@@ -0,0 +1 @@
+../../action_plugins

playbooks/byo/library

@@ -0,0 +1 @@
+../../library

playbooks/byo/openshift-cfme/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/openshift-checks/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/openshift-checks/certificate_expiry/action_plugins

@@ -0,0 +1 @@
+../../../../action_plugins

playbooks/byo/openshift-cluster/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/openshift-cluster/upgrades/action_plugins

@@ -0,0 +1 @@
+../../../../action_plugins

playbooks/byo/openshift-cluster/upgrades/docker/action_plugins

@@ -0,0 +1 @@
+../../../../../action_plugins

playbooks/byo/openshift-cluster/upgrades/v3_3/action_plugins

@@ -0,0 +1 @@
+../../../../../action_plugins

playbooks/byo/openshift-cluster/upgrades/v3_4/action_plugins

@@ -0,0 +1 @@
+../../../../../action_plugins

playbooks/byo/openshift-cluster/upgrades/v3_5/action_plugins

@@ -0,0 +1 @@
+../../../../../action_plugins

playbooks/byo/openshift-cluster/upgrades/v3_6/action_plugins

@@ -0,0 +1 @@
+../../../../../action_plugins

playbooks/byo/openshift-cluster/upgrades/v3_7/action_plugins

@@ -0,0 +1 @@
+../../../../../action_plugins

playbooks/byo/openshift-etcd/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/openshift-glusterfs/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/openshift-master/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/openshift-node/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/byo/openshift-preflight/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/common/action_plugins

@@ -0,0 +1 @@
+../../action_plugins

playbooks/common/openshift-cfme/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/common/openshift-checks/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/common/openshift-cluster/action_plugins

@@ -0,0 +1 @@
+../../../action_plugins

playbooks/common/openshift-cluster/initialize_facts.yml

@@ -88,7 +88,7 @@
     - not l_is_atomic | bool
     block:
     - name: Ensure openshift-ansible installer package deps are installed
-      package:
+      openshift_package:
         name: "{{ item }}"
         state: present
       with_items:
@@ -98,7 +98,7 @@
       - yum-utils
 
     - name: Ensure various deps for running system containers are installed
-      package:
+      openshift_package:
         name: "{{ item }}"
         state: present
       with_items:

playbooks/common/openshift-cluster/redeploy-certificates/action_plugins

@@ -0,0 +1 @@
+../../../../action_plugins

playbooks/common/openshift-cluster/upgrades/action_plugins

@@ -0,0 +1 @@
+../../../../action_plugins

playbooks/common/openshift-cluster/upgrades/docker/action_plugins

@@ -0,0 +1 @@
+../../../../../action_plugins