Add tactical work around for traefik

Add tactical work around for traefik readyness. Traefik takes
sometime to process changes to load balancer configuration.
Traefik has an api but it is turned off by default and currently
the charm does not expose a way to enable it. So, this change
looks at the traefik config on disk to check if it has the
correct number if units.

zaza/openstack/charm_tests/tempest/tests.py

@@ -17,6 +17,7 @@
 import logging
 import os
 import subprocess
+import yaml
 
 import zaza
 import zaza.charm_lifecycle.utils
@@ -184,6 +185,39 @@ def wait_for_dead_units(self, application_name, dead_count):
                       if ustatus.agent_status.life == 'dying']
         assert len(dead_units) == dead_count
 
+    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
+                    reraise=True, stop=tenacity.stop_after_attempt(8))
+    def wait_for_traefik(self, application_name):
+        """Wait for traefk to finish processing lb changes."""
+        logging.warning(
+            "Waiting for traefik to process changes. This is a temporary "
+            "workaround and should be removed when there is a way to "
+            "determine when traefik has processed all requests")
+        units_count = len(zaza.model.get_units(application_name))
+        container_cmd = (
+            "cat /opt/traefik/juju/juju_ingress_ingress_*_{}.yaml").format(
+                application_name)
+        container_name = "traefik"
+        for unit in zaza.model.get_units("traefik"):
+            config = subprocess.check_output([
+                "juju",
+                "ssh",
+                "-m", zaza.model.get_juju_model(),
+                "--container",
+                container_name,
+                unit.entity_id,
+                container_cmd]).decode()
+            service_config = yaml.safe_load(config)
+            loadBalancers = [
+                lb['loadBalancer']
+                for lb in service_config['http']['services'].values()]
+            assert len(loadBalancers) == 1
+            unit_count_in_lb = len(loadBalancers[0]['servers'])
+            logging.info("Traefik LB server count: {} unit count: {}".format(
+                unit_count_in_lb,
+                units_count))
+            assert unit_count_in_lb == units_count
+
     def run(self):
         """Run tempest tests as specified in tests/tests.yaml.
 
@@ -197,6 +231,7 @@ def run(self):
         render_tempest_config_keystone_v3 = tenacity.retry(
             wait=tenacity.wait_fixed(10), stop=tenacity.stop_after_attempt(3)
         )(tempest_utils.render_tempest_config_keystone_v3)
+        self.wait_for_traefik(self.application_name)
         zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints()
         render_tempest_config_keystone_v3(minimal=True)
         if not super().run():
@@ -208,6 +243,7 @@ def run(self):
         zaza.model.block_until_all_units_idle()
         logging.info("Wait for status ready ...")
         zaza.model.wait_for_application_states(states=self.expected_statuses)
+        self.wait_for_traefik(self.application_name)
         zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints()
         render_tempest_config_keystone_v3(minimal=True)
         if not super().run():
@@ -222,6 +258,7 @@ def run(self):
         zaza.model.block_until_all_units_idle()
         logging.info("Wait for status ready ...")
         zaza.model.wait_for_application_states(states=self.expected_statuses)
+        self.wait_for_traefik(self.application_name)
         zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints()
         render_tempest_config_keystone_v3(minimal=True)
         return super().run()