Build: disable builds after 25 consecutive failed builds on default version

docs/user/builds.rst

@@ -142,3 +142,35 @@ Our build limits are:
       If your business is hitting build limits hosting documentation on Read the Docs,
       please consider :doc:`Read the Docs for Business </commercial/index>`
       which has options for additional build resources.
+
+
+.. _builds:automatic-disabling:
+
+Automatic disabling of builds
+-----------------------------
+
+To reduce resource consumption and improve build queue times for all users,
+Read the Docs will automatically disable builds for projects that have too many consecutive failed builds on their default version.
+
+When a project has **50 consecutive failed builds** on its default version,
+we will:
+
+* Attach a notification to the project explaining the situation
+* Disable builds for the project by setting ``skip=True``
+
+This helps ensure that projects with persistent build issues don't consume resources that could be used by active projects.
+
+.. note::
+
+   This only applies to the default version of a project.
+   Builds on other versions (branches, tags, pull requests) are not counted towards this limit.
+
+**How to re-enable builds:**
+
+If your project has been disabled due to consecutive build failures,
+you'll need to:
+
+1. Fix the underlying build issues in your repository
+2. Contact support to have your project re-enabled
+
+Once re-enabled, we recommend triggering a manual build to verify that your documentation builds successfully.

readthedocs/builds/signals_receivers.py

@@ -4,11 +4,28 @@
 NOTE: Done in a separate file to avoid circular imports.
 """
 
+import structlog
+from django.conf import settings
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 
 from readthedocs.builds.models import Build
+from readthedocs.builds.signals import build_complete
+from readthedocs.notifications.models import Notification
 from readthedocs.projects.models import Project
+from readthedocs.projects.notifications import (
+    MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES,
+)
+
+
+log = structlog.get_logger(__name__)
+
+
+# Number of consecutive failed builds on the default version
+# before we disable the project.
+RTD_BUILDS_MAX_CONSECUTIVE_FAILURES = getattr(
+    settings, "RTD_BUILDS_MAX_CONSECUTIVE_FAILURES", 50
+)
 
 
 @receiver(post_save, sender=Build)
@@ -18,3 +35,98 @@ def update_latest_build_for_project(sender, instance, created, **kwargs):
         Project.objects.filter(pk=instance.project_id).update(
             latest_build=instance,
         )
+
+
+@receiver(build_complete, sender=Build)
+def disable_project_on_consecutive_failed_builds(sender, build, **kwargs):
+    """
+    Disable a project if it has too many consecutive failed builds on the default version.
+
+    When a project has 50+ consecutive failed builds on the default version,
+    we attach a notification to the project and disable builds (skip=True).
+    This helps reduce resource consumption from projects that are not being monitored.
+    """
+    # Build is a dict coming from the task, not a Build instance
+    if not isinstance(build, dict):
+        return
+
+    # Only check on failed builds
+    if build.get("success"):
+        return
+
+    project_id = build.get("project")
+    version_slug = build.get("version_slug")
+
+    if not project_id or not version_slug:
+        return
+
+    try:
+        project = Project.objects.get(pk=project_id)
+    except Project.DoesNotExist:
+        return
+
+    # Only check for the default version
+    if version_slug != project.get_default_version():
+        return
+
+    # Skip if the project is already disabled
+    if project.skip:
+        return
+
+    # Count consecutive failed builds on the default version
+    consecutive_failed_builds = _count_consecutive_failed_builds(project, version_slug)
+
+    if consecutive_failed_builds >= RTD_BUILDS_MAX_CONSECUTIVE_FAILURES:
+        log.info(
+            "Disabling project due to consecutive failed builds.",
+            project_slug=project.slug,
+            version_slug=version_slug,
+            consecutive_failed_builds=consecutive_failed_builds,
+        )
+
+        # Disable the project
+        project.skip = True
+        project.save()
+
+        # Attach notification to the project
+        Notification.objects.add(
+            message_id=MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES,
+            attached_to=project,
+            dismissable=False,
+            format_values={
+                "consecutive_failed_builds": consecutive_failed_builds,
+            },
+        )
+
+
+def _count_consecutive_failed_builds(project, version_slug):
+    """
+    Count the number of consecutive failed builds on a version.
+
+    We look at the most recent builds (up to the limit) and count
+    how many have failed consecutively from the most recent build.
+    """
+    from readthedocs.builds.constants import BUILD_STATE_FINISHED
+
+    # Get the most recent finished builds for this version.
+    # We fetch a few extra builds beyond the threshold to ensure we can
+    # accurately count consecutive failures even if there's a success just after.
+    builds = (
+        Build.objects.filter(
+            project=project,
+            version_slug=version_slug,
+            state=BUILD_STATE_FINISHED,
+        )
+        .order_by("-date")
+        .values_list("success", flat=True)[: RTD_BUILDS_MAX_CONSECUTIVE_FAILURES + 1]
+    )
+
+    consecutive_failures = 0
+    for success in builds:
+        if not success:
+            consecutive_failures += 1
+        else:
+            # First successful build breaks the streak
+            break
+
+    return consecutive_failures