feat: send only one GrafanaSource to grafana (#379)

* feat: send only one GrafanaSource to grafana

* fix tests

* chore: add comment and add itest

* chore: lint

* fix: static

* fix: itest

* fix: comment

* test: add traefik to itest

* test: enhance docstring

* fix: remove try except

* fix: index

* fix: lint

* fix: test

---------

Co-authored-by: Sina P <[email protected]>
Co-authored-by: Sina <[email protected]>

Author: lucabello

src/charm.py

@@ -59,6 +59,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 @dataclass
 class TLSConfig:
     """TLS configuration received by the charm over the `certificates` relation."""
@@ -67,6 +68,7 @@ class TLSConfig:
     ca_cert: str
     private_key: str
 
+
 @trace_charm(
     tracing_endpoint="_charm_tracing_endpoint",
     server_cert="_charm_tracing_ca_cert",
@@ -143,7 +145,8 @@ def __init__(self, *args):
         self.grafana_source_provider = GrafanaSourceProvider(
             charm=self,
             source_type="alertmanager",
-            source_url=self._external_url,
+            source_url=self.ingress.url or self._service_url,
+            is_ingress_per_app=True, # We want only one alertmanager datasource (unit) to be listed in grafana.
             refresh_event=[
                 self.ingress.on.ready,
                 self.ingress.on.revoked,
@@ -280,9 +283,7 @@ def set_ports(self):
 
     @property
     def _catalogue_item(self) -> CatalogueItem:
-        api_endpoints = {
-            "Alerts": "/api/v2/alerts"
-        }
+        api_endpoints = {"Alerts": "/api/v2/alerts"}
 
         return CatalogueItem(
             name="Alertmanager",
@@ -294,7 +295,9 @@ def _catalogue_item(self) -> CatalogueItem:
                 "the configured receiver(s)."
             ),
             api_docs="https://github.com/prometheus/alertmanager/blob/main/api/v2/openapi.yaml",
-            api_endpoints={key: f"{self._external_url}{path}" for key, path in api_endpoints.items()},
+            api_endpoints={
+                key: f"{self._external_url}{path}" for key, path in api_endpoints.items()
+            },
         )
 
     @property
@@ -570,8 +573,7 @@ def _on_update_status(self, _):
         try:
             status = self.alertmanager_workload.api.status()
             logger.info(
-                "alertmanager %s is up and running (uptime: %s); "
-                "cluster mode: %s, with %d peers",
+                "alertmanager %s is up and running (uptime: %s); cluster mode: %s, with %d peers",
                 status["versionInfo"]["version"],
                 status["uptime"],
                 status["cluster"]["status"],
@@ -641,6 +643,19 @@ def _internal_url(self) -> str:
         """Return the fqdn dns-based in-cluster (private) address of the alertmanager api server."""
         return f"{self._scheme}://{self._fqdn}:{self._ports.api}"
 
+    @property
+    def _service_url(self) -> str:
+        """Return the FQDN DNS-based in-cluster (private) address of the service for Alertmanager.
+
+        Since our goal is to ensure that we only send one datasource to Grafana when we have multiple units, we can't use the socket FQDN because that would include the AM unit e.g. `http://am-0.am-endpoints.otel.svc.cluster.local:9093`.
+        The service URL as defined will remove the pod unit so (when ingress missing) the request goes to the K8s service at: http://am-endpoints.otel.svc.cluster.local:9093
+        The service will then load balance between the units.
+        This assumes that the FQDN is the interal FQDN for the socket and that the pod unit is always on the left side of the first ".". If those change, this code will need to be updated.
+        """
+        fqdn = self._fqdn.split(".", 1)[-1]
+
+        return f"{self._scheme}://{fqdn}:{self._ports.api}"
+
     @property
     def _external_url(self) -> str:
         """Return the externally-reachable (public) address of the alertmanager api server."""

tests/integration/conftest.py

@@ -10,6 +10,7 @@
 from datetime import datetime
 from pathlib import Path
 
+import juju.utils
 import pytest
 from pytest_operator.plugin import OpsTest
 
@@ -94,3 +95,13 @@ async def setup_env(ops_test: OpsTest):
 @pytest.fixture(scope="module")
 def temp_dir(tmp_path_factory):
     return tmp_path_factory.mktemp("data")
+
+@pytest.fixture(scope="module", autouse=True)
+def patch_pylibjuju_series_2404():
+    juju.utils.ALL_SERIES_VERSIONS["noble"] = "24.04"
+    juju.utils.UBUNTU_SERIES["noble"] = "24.04"
+
+    yield
+
+    del juju.utils.ALL_SERIES_VERSIONS["noble"]
+    del juju.utils.UBUNTU_SERIES["noble"]

tests/integration/helpers.py

@@ -11,7 +11,10 @@
 from typing import Dict, Optional, Tuple
 from urllib.parse import urlparse
 
+import requests
+from juju.unit import Unit
 from pytest_operator.plugin import OpsTest
+from requests.auth import HTTPBasicAuth
 
 logger = logging.getLogger(__name__)
 
@@ -171,3 +174,54 @@ async def curl(ops_test: OpsTest, *, cert_dir: str, cert_path: str, ip_addr: str
         "non-zero return code means curl encountered a >= 400 HTTP code"
     )
     return stdout
+
+async def grafana_password(ops_test: OpsTest, app_name: str) -> str:
+    """Get the admin password. Memoize it to reduce turnaround time.
+
+    Args:
+        ops_test: pytest-operator plugin
+        app_name: string name of application
+
+    Returns:
+        admin password as a string
+    """
+    leader: Optional[Unit] = None
+    for unit in ops_test.model.applications[app_name].units:  # type: ignore
+        is_leader = await unit.is_leader_from_status()
+        if is_leader:
+            leader = unit
+            break
+
+    assert leader
+    action = await leader.run_action("get-admin-password")
+    action = await action.wait()
+    return action.results["admin-password"]
+
+async def grafana_datasources(ops_test: OpsTest, app_name: str) -> "list[dict]":
+    """Get the datasources configured in Grafana.
+
+    A sample response from Grafana's /api/datasources endpoint is a list of datasources, similar to below.
+
+    [{"id":1,"uid":"ABC","orgId":1,"name":"<name>",
+    "type":"alertmanager","typeName":"Alertmanager",
+    "typeLogoUrl":"public/app/plugins/datasource/alertmanager/img/logo.svg","access":"proxy",
+    "url":"<AM-address>","user":"","database":"","basicAuth":false,"isDefault":false,
+    "jsonData":{"implementation":"prometheus","timeout":300},"readOnly":true}}, ...]
+
+    Args:
+        ops_test: pytest-operator plugin
+        app_name: string name of application
+    Returns:
+        number of datasources as an integer
+    """
+    address = await get_unit_address(ops_test, app_name, 0)
+    url = f"http://{address}:3000/api/datasources"
+
+    admin_password = await grafana_password(ops_test, app_name)
+    response = requests.get(
+        url,
+        auth=HTTPBasicAuth("admin", admin_password),
+    )
+    response.raise_for_status()
+    datasources = response.json()
+    return datasources

tests/integration/test_grafana_source.py

@@ -0,0 +1,58 @@
+import asyncio
+import logging
+from pathlib import Path
+
+import pytest
+import yaml
+from helpers import grafana_datasources
+from pytest_operator.plugin import OpsTest
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+# pyright: reportAttributeAccessIssue = false
+# pyright: reportOptionalMemberAccess = false
+
+logger = logging.getLogger(__name__)
+
+METADATA = yaml.safe_load(Path("./charmcraft.yaml").read_text())
+app_name = METADATA["name"]
+resources = {"alertmanager-image": METADATA["resources"]["alertmanager-image"]["upstream-source"]}
+
+"""We need to ensure that, even if there are multiple units for Alertmanager, only one is shown as a datasouce in Grafana.
+We use this test to simulate multiple units of Alertmanager, and then check that only the leader has the key `grafana_source_host` written to relation data with Grafana.
+"""
+
+@pytest.mark.abort_on_fail
+async def test_build_and_deploy(ops_test: OpsTest, charm_under_test):
+    """Build the charm-under-test, deploy the charm from charmhub, and upgrade from path."""
+    await asyncio.gather(
+        ops_test.model.deploy(charm_under_test, "am", resources=resources, trust=True, num_units=2),
+        ops_test.model.deploy("grafana-k8s", "grafana", channel="2/edge", trust=True),
+    )
+
+    await ops_test.model.add_relation("grafana:grafana-source", "am")
+    await ops_test.model.wait_for_idle(apps=["am", "grafana"], status="active")
+
+@retry(wait=wait_fixed(10), stop=stop_after_attempt(6))
+async def test_grafana_datasources(ops_test: OpsTest):
+    # We have 2 units of Alertmanager, but only one datasource should be shown as a Grafana source.
+    datasources = await grafana_datasources(ops_test, "grafana")
+    assert len(datasources) == 1
+
+    # The datasource URL should point to the service, not to a specific pod unit.
+    # This check is safe, because we name the application `am` and we're not using TLS, so the service will always start with `http://am-endpoints`.
+    assert datasources[0]["url"].startswith("http://am-endpoints")
+
+@pytest.mark.abort_on_fail
+async def test_deploy_and_integrate_traefik(ops_test: OpsTest):
+    """Build the charm-under-test, deploy the charm from charmhub, and upgrade from path."""
+    await ops_test.model.deploy("traefik-k8s", "traefik", channel="edge", trust=True)
+
+    await ops_test.model.add_relation("traefik:ingress", "am")
+    await ops_test.model.wait_for_idle(apps=["am", "grafana", "traefik"], status="active")
+
+async def test_grafana_datasources_when_ingress_available(ops_test: OpsTest):
+    # We have 2 units of Alertmanager, but only one datasource should be shown as a Grafana source.
+    datasources = await grafana_datasources(ops_test, "grafana")
+    assert len(datasources) == 1
+
+    assert "am-endpoints" not in datasources[0]["url"]