Sync upstream/integration (2940a01) -> release/4 2026-01-14

.github/workflows/test_deactivatorg.yaml → .github/workflows/test_bailiff.yml

@@ -1,11 +1,11 @@
-name: Build deactivatorg test image and run tests on it
-run-name: Build deactivatorg test image and run tests on it - started by ${{ github.actor }}
+name: Build bailiff test image and run tests on it
+run-name: Build bailiff test image and run tests on it - started by ${{ github.actor }}
 permissions: read-all
 on:
   pull_request:
     types: [opened, synchronize]
     paths:
-    - 'docker_images/deactivatorg/**'
+    - 'docker_images/bailiff/**'
   workflow_dispatch:
 
 
@@ -25,6 +25,6 @@ jobs:
           username: ${{ env.username }}
           password: ${{ env.password }}
       - name: Build image
-        run: bash -x build.sh deactivatorg build
+        run: bash -x build.sh bailiff build
       - name: Build test image and run tests
-        run: bash -x docker_images/run_test.sh deactivatorg
+        run: bash -x docker_images/run_test.sh bailiff

.github/workflows/test_subspector.yml

@@ -0,0 +1,30 @@
+name: Build subspector test image and run tests on it
+run-name: Build subspector test image and run tests on it - started by ${{ github.actor }}
+permissions: read-all
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+    - 'subspector/**'
+  workflow_dispatch:
+
+
+jobs:
+  build_image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Login to Docker Hub (optional)
+        env:
+          username: ${{ secrets.EXT_DOCKER_LOGIN }}
+          password: ${{ secrets.EXT_DOCKER_TOKEN }}
+        if: ${{ env.username != '' && env.password != ''}}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.username }}
+          password: ${{ env.password }}
+      - name: Build image
+        run: bash -x build.sh subspector build
+      - name: Build test image and run tests
+        run: bash -x subspector/run_test.sh

.github/workflows/test_subsyncer.yml

@@ -0,0 +1,30 @@
+name: Build subsyncer test image and run tests on it
+run-name: Build subsyncer test image and run tests on it - started by ${{ github.actor }}
+permissions: read-all
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+    - 'docker_images/subsyncer/**'
+  workflow_dispatch:
+
+
+jobs:
+  build_image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Login to Docker Hub (optional)
+        env:
+          username: ${{ secrets.EXT_DOCKER_LOGIN }}
+          password: ${{ secrets.EXT_DOCKER_TOKEN }}
+        if: ${{ env.username != '' && env.password != ''}}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.username }}
+          password: ${{ env.password }}
+      - name: Build image
+        run: bash -x build.sh subsyncer build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh subsyncer

bumiworker/bumiworker/modules/archive/obsolete_images.py → bumiworker/bumiworker/modules/archive/snapshots_with_non_used_images.py

@@ -5,17 +5,20 @@
 
 from bumiworker.bumiworker.consts import ArchiveReason
 from bumiworker.bumiworker.modules.base import ArchiveBase
-from bumiworker.bumiworker.modules.recommendations.obsolete_images import (
-    ObsoleteImages as ObsoleteImagesRecommendation, SUPPORTED_CLOUD_TYPES)
+from bumiworker.bumiworker.modules.recommendations.snapshots_with_non_used_images import (
+    SnapshotsWithNonUsedImages as SnapshotsWithNonUsedImagesRecommendation,
+    SUPPORTED_CLOUD_TYPES
+)
 
 LOG = logging.getLogger(__name__)
 
 
-class ObsoleteImages(ArchiveBase, ObsoleteImagesRecommendation):
+class SnapshotsWithNonUsedImages(ArchiveBase,
+                                 SnapshotsWithNonUsedImagesRecommendation):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.reason_description_map[ArchiveReason.RECOMMENDATION_APPLIED] = (
-            'image deleted')
+            'snapshot deleted')
 
     @property
     def supported_cloud_types(self):
@@ -35,15 +38,14 @@ def _get(self, previous_options, optimizations, cloud_accounts_map,
         starting_point = datetime.now(timezone.utc) - timedelta(
             days=days_threshold)
 
-        images_map = self._get_images_map(cloud_accounts_map.values(), starting_point)
+        images_map = self._get_images_map(cloud_accounts_map.values())
         used_image_ids = self.mongo_client.restapi.resources.distinct(
             'meta.image_id', {
                 'cloud_account_id': {'$in': list(cloud_accounts_map.keys())},
                 'resource_type': 'Instance',
                 'meta.image_id': {'$in': list(images_map.keys())},
                 '_last_seen_date': {'$gte': self.timestamp_to_day_start(
-                    starting_point.timestamp())},
-                'first_seen': {'$gte': starting_point.timestamp()}
+                    starting_point.timestamp())}
             }
         )
         for cloud_account_id, optimizations_ in account_optimizations_map.items():
@@ -53,12 +55,24 @@ def _get(self, previous_options, optimizations, cloud_accounts_map,
                         optimization, ArchiveReason.CLOUD_ACCOUNT_DELETED)
                     result.append(optimization)
                 continue
+            resource_ids = [x['resource_id'] for x in optimizations_]
+            snapshots = [
+                x['_id'] for x in self.mongo_client.restapi.resources.find(
+                    {'_id': {'$in': resource_ids}, 'active': True})
+            ]
             for optimization in optimizations_:
-                resource_id = optimization['cloud_resource_id']
-                if resource_id not in images_map:
+                resource_id = optimization['resource_id']
+                images = optimization['images']
+                if resource_id not in snapshots:
+                    # snapshot deleted
                     self._set_reason_properties(
                         optimization, ArchiveReason.RECOMMENDATION_APPLIED)
-                elif resource_id in used_image_ids:
+                elif any(x in used_image_ids for x in images):
+                    # image used for instance
+                    self._set_reason_properties(
+                        optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT)
+                elif all(x not in images_map for x in images):
+                    # image deleted
                     self._set_reason_properties(
                         optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT)
                 else:
@@ -69,5 +83,5 @@ def _get(self, previous_options, optimizations, cloud_accounts_map,
 
 
 def main(organization_id, config_client, created_at, **kwargs):
-    return ObsoleteImages(
+    return SnapshotsWithNonUsedImages(
         organization_id, config_client, created_at).get()

bumiworker/bumiworker/modules/recommendations/s3_intelligent_tiering.py

@@ -1,6 +1,6 @@
 import logging
 from collections import defaultdict
-from concurrent.futures.thread import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, Dict, List, Tuple
 from datetime import datetime, date, timedelta
 
@@ -205,7 +205,7 @@ def _display_name(storage_type):
             })
         metrics = aws.get_cloud_watch_metric_data(
             region, metric_queries, today,
-            today - timedelta(days=7))
+            today - timedelta(days=2))
         for md in metrics.get("MetricDataResults", []):
             values = md.get("Values") or []
             if not values:
@@ -228,14 +228,21 @@ def _candidates_and_savings(self,
         """
         buckets_data = {}
         result = []
-        with ThreadPoolExecutor(max_workers=50) as executor:
+        if not region_candidates:
+            return result
+        cw_clients = {
+            region: aws.session.client("cloudwatch", region_name=region)
+            for region in region_candidates
+        }
+        max_workers = min(20, sum(len(v) for v in region_candidates.values()))
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
             futures = []
-            for region, candidates in region_candidates.items():
-                s3_client = aws.session.client("s3", region_name=region)
-                for cand in candidates:
+            for region, buckets in region_candidates.items():
+                cloudwatch = cw_clients[region]
+                for bucket in buckets:
                     futures.append(executor.submit(
-                        aws.get_bucket_storage_info, s3_client, cand, ))
-            for f in futures:
+                        aws.get_bucket_storage_info, cloudwatch, bucket, ))
+            for f in as_completed(futures):
                 res = f.result()
                 if res:
                     buckets_data.update(res)