Removed requests python dependency in cf-bottom

Ticket: CFE-4188
Signed-off-by: Victor Moene <victor.moene@northern.tech>

Author: victormlg

requirements.txt

@@ -1 +0,0 @@
-requests==2.32.4

tom/changelog.py

@@ -1,6 +1,5 @@
 import re
 import os
-import requests
 import subprocess
 import datetime
 import hashlib
@@ -125,7 +124,7 @@ def generate_changelog_in_repo(self, branch, repo):
             return False
 
         # find old and new versions
-        (header, old_version, old_changelog) = self.split_changelog_into_parts(
+        header, old_version, old_changelog = self.split_changelog_into_parts(
             changelog_filename, repo
         )
         if header != "":

tom/dependencies.py

@@ -1,6 +1,5 @@
 import re
 import json
-import requests
 import collections
 import datetime
 import hashlib
@@ -134,19 +133,23 @@ def checkfile(self, url, sha256=False):
             True, False, or sha256 of a linked file
         """
         log.debug("checking URL: " + url)
+
+        is_head = not sha256 and url.startswith("http")
+        method = "HEAD" if is_head else "GET"
         try:
-            if not sha256 and url.startswith("http"):
-                log.debug("testing with HEAD")
-                r = requests.head(url)
-                return r.status_code >= 200 and r.status_code < 300
-            else:
+            req = urllib.request.Request(url, method=method)
+            with urllib.request.urlopen(req) as f:
+                if is_head:
+                    log.debug("testing with HEAD")
+                    return True
+
                 log.debug("getting whole file")
                 m = hashlib.sha256()
-                with urllib.request.urlopen(url) as f:
+
+                data = f.read(4096)
+                while data:
+                    m.update(data)
                     data = f.read(4096)
-                    while data:
-                        m.update(data)
-                        data = f.read(4096)
                 return m.hexdigest()
         except:
             return False
@@ -222,7 +225,8 @@ def get_version_from_monitoring(self, dep):
         id = self.monitoring_ids[dep]
         url = "https://release-monitoring.org/api/v2/versions/?project_id={}".format(id)
         try:
-            data = requests.get(url).json()
+            with urllib.request.urlopen(url) as r:
+                data = json.loads(r)
         except:
             raise ReleaseMonitoringException(
                 "Failed to do a request to release-monitoring.org website"
@@ -255,7 +259,7 @@ def get_current_version(self, dep):
         dist_file = self.buildscripts.get_file(dist_file_path)
         dist_file = dist_file.strip()
         old_filename = re.sub(".* ", "", dist_file)
-        (old_version, separator) = self.extract_version_from_filename(dep, old_filename)
+        old_version, separator = self.extract_version_from_filename(dep, old_filename)
         return old_version
 
     def patch_spec_file(self, spec_file_path, old_version, new_version):
@@ -283,7 +287,7 @@ def update_single_dep(self, dep):
         source_file = source_file.strip()
         old_filename = re.sub(".* ", "", dist_file)
         old_url = "{}{}".format(source_file, old_filename)
-        (old_version, separator) = self.extract_version_from_filename(dep, old_filename)
+        old_version, separator = self.extract_version_from_filename(dep, old_filename)
         new_version = self.get_version_from_monitoring(dep)
         if not new_version:
             log.warning(

tom/github.py

@@ -1,12 +1,12 @@
 import random
-import requests
 import re
 import os
 import sys
 import json
 import datetime
+import urllib.error
+import urllib.request
 import logging as log
-import requests
 from copy import copy
 
 from tom.utils import pretty, write_json
@@ -36,15 +36,16 @@ def get(self, path):
         if path in self.get_cache:
             log.debug("Found in cache")
             return self.get_cache[path]
-        r = requests.get(path, headers=self.headers)
-        log.debug("RESPONSE {}".format(r.status_code))
 
-        if not (200 <= r.status_code < 300):
-            sys.exit("Non-success API response {} for '{}'".format(r.status_code, path))
+        req = urllib.request.Request(path, headers=self.headers)
+        try:
+            with urllib.request.urlopen(req) as r:
+                data = json.load(r)
+                log.debug("RESPONSE {}".format(r.code))
+        except urllib.error.HTTPError as e:
+            sys.exit("Non-success API response {} for '{}'".format(e.code, path))
 
-        assert r.status_code >= 200 and r.status_code < 300
-        data = r.json()
-        log.debug(pretty(data))
+        log.debug(json.dumps(data))
         self.get_cache[path] = data
         if (
             not isinstance(data, list)
@@ -74,12 +75,19 @@ def post(self, path, data, check_status_code=True):
             return None
         self.api_log("POST {} {}".format(path, data))
         path = self.path(path)
-        r = requests.post(path, headers=self.headers, json=data)
-        log.debug("RESPONSE {}".format(r.status_code))
-        if check_status_code:
-            assert r.status_code >= 200 and r.status_code < 300, r.text
-        data = r.json()
-        log.debug(pretty(data))
+
+        json_data = json.dumps(data).encode("utf-8")
+        req = urllib.request.Request(path, headers=self.headers, data=json_data)
+        try:
+            with urllib.request.urlopen(req) as r:
+                log.debug("RESPONSE {}".format(r.status_code))
+                data = json.load(r)
+                log.debug(r.read().decode())
+        except urllib.error.HTTPError as e:
+            log.error(e.headers)
+            log.error(e.read().decode())
+            raise AssertionError("HTTP response {} from GitHub".format(e.code))
+
         return data
 
     def create_pr(
@@ -388,7 +396,7 @@ def create_prs_from_slack(self):
         last_branch = self.find_last_branch_in_repo(username, repo)
         log.info("last branch: " + last_branch)
         # now, try to find a parent for it.
-        (parent_repo, parent_branch) = self.find_parent(username, repo, last_branch)
+        parent_repo, parent_branch = self.find_parent(username, repo, last_branch)
         log.info("parent branch: " + parent_branch)
         pr_text = self.github.create_pr(
             parent_repo, parent_branch, username, last_branch, last_branch + " PR", ""

tom/jenkins.py

@@ -1,11 +1,14 @@
 import logging as log
 from time import sleep
-import requests
-from requests.auth import HTTPBasicAuth
 from tom.utils import pretty
 import os
+import base64
+import json
 from typing import Dict
 
+import urllib.request
+import urllib.error
+
 
 class Jenkins:
     def __init__(self, url, job, secrets, username):
@@ -20,7 +23,7 @@ def __init__(self, url, job, secrets, username):
         self.crumb = crumb
         self.username = username
 
-        self.auth = HTTPBasicAuth(user, token)
+        self.auth = base64.b64encode(("%s:%s" % (user, token)).encode()).decode()
 
         self.headers = {}
         if crumb:
@@ -34,17 +37,20 @@ def post(self, path, data):
         if os.getenv("TOM") == "PASSIVE":
             print("Would post: " + path)
             return None
-        r = requests.post(path, data=data, headers=self.headers, auth=self.auth)
-        if not (200 <= r.status_code < 300):
-            log.error("Unexpected HTTP response from Jenkins: {}".format(r.status_code))
-            log.error(str(r.headers))
-            log.error(str(r.text))
-            raise AssertionError("HTTP response {} from Jenkins".format(r.status_code))
-
+        json_data = json.dumps(data).encode("utf-8")
+        req = urllib.request.Request(path, data=json_data, headers=self.headers)
+        req.add_header("Authorization", "Basic %s" % self.auth)
         try:
-            return r.headers, r.json()
-        except:
-            return r.headers, r.text
+            with urllib.request.urlopen(req) as r:
+                try:
+                    parsed = json.load(r)
+                    return r.headers, parsed
+                except:
+                    return r.headers, r.read().decode()
+        except urllib.error.HTTPError as e:
+            log.error(e.headers)
+            log.error(e.read().decode())
+            raise AssertionError("HTTP response {} from Jenkins".format(e.code))
 
     def trigger(
         self,
@@ -144,9 +150,16 @@ def wait_for_queue(self, url):
         while "executable" not in queue_item:
             log.info("Waiting for jenkins build in queue")
             sleep(1)
-            r = requests.get(url + "api/json", headers=self.headers, auth=self.auth)
-            assert r.status_code >= 200 and r.status_code < 300
-            queue_item = r.json()
+            req = urllib.request.Request("{}api/json".format(url), headers=self.headers)
+            req.add_header("Authorization", "Basic %s" % self.auth)
+            try:
+                with urllib.request.urlopen(req) as r:
+                    queue_item = json.load(r)
+            except urllib.error.HTTPError as e:
+                log.error("Unexpected HTTP response from Jenkins: {}".format(e.code))
+                log.error(e.headers)
+                log.error(e.read().decode())
+                raise AssertionError("HTTP response {} from Jenkins".format(e.code))
         log.debug(pretty(queue_item))
 
         num = queue_item["executable"]["number"]

tom/packages.py

@@ -1,8 +1,9 @@
 import re
 import json
-import requests
 import collections
 import datetime
+import urllib.request
+import urllib.error
 from tom.git import GitRepo
 
 
@@ -51,6 +52,18 @@ def is_branch_or_version(string):
     return None
 
 
+def fetch_json(url):
+    try:
+        with urllib.request.urlopen(url) as r:
+            return json.load(r)
+    except urllib.error.HTTPError as e:
+        raise URLDownloadFailureException(
+            "failed to download %s, return code %d" % (url, e.code)
+        )
+    except json.decoder.JSONDecodeError as e:
+        raise JSONParsingError("file %s is not a valid JSON" % url) from e
+
+
 class PackageMapper:
     """Class responsible for updating packages_mapping.json file
     Currently it's tailored for cfengine needs.
@@ -125,18 +138,8 @@ def run(self, inputs):
                 releases_url = (
                     "https://cfengine.com/release-data/%s/releases.json" % product
                 )
-                releases_request = requests.get(releases_url)
-                if not releases_request.ok:
-                    raise URLDownloadFailureException(
-                        "failed to download %s, return code %d"
-                        % (releases_url, releases_request.status_code)
-                    )
-                try:
-                    releases_data = releases_request.json()
-                except json.decoder.JSONDecodeError as e:
-                    raise JSONParsingError(
-                        "file %s is not a valid JSON" % releases_url
-                    ) from e
+                releases_data = fetch_json(releases_url)
+
                 if "releases" not in releases_data:
                     raise JSONStructureError('no "releases" in %s JSON' % releases_url)
                 if value_type == "version":
@@ -233,16 +236,7 @@ def run(self, inputs):
 
     def collect_packages(self, url):
         """Given a release URL, returns a dict where keys are platform names, and values are {'url': 'http...'}"""
-        release_request = requests.get(url)
-        if not release_request.ok:
-            raise URLDownloadFailureException(
-                "failed to download %s, return code %d"
-                % (url, release_request.status_code)
-            )
-        try:
-            release_data = release_request.json()
-        except json.decoder.JSONDecodeError as e:
-            raise JSONParsingError("file %s is not a valid JSON" % url) from e
+        release_data = fetch_json(url)
         if "artifacts" not in release_data:
             raise JSONStructureError('no "artifacts" in %s JSON' % url)
         # release_data['artifacts'] is a dictionary (called 'table'), each element is a list

tom/slack.py

@@ -3,8 +3,9 @@
 import re
 import sys
 import json
-import requests
 import logging as log
+import urllib.request
+import urllib.error
 from tom.utils import pretty
 
 
@@ -36,13 +37,25 @@ def post(self, url, data={}):
             url = self.api(url)
         if not "token" in data:
             data["token"] = self.bot_token
-        r = requests.post(url, data)
-        assert r.status_code >= 200 and r.status_code < 300
+
+        json_data = json.dumps(data).encode("utf-8")
+        req = urllib.request.Request(
+            url,
+            data=json_data,
+            method="POST",
+            headers={"Content-Type": "application/json"},
+        )
         try:
-            log.debug(pretty(r.json()))
-            return r.json()
-        except:
-            log.debug(pretty(r.text))
+            with urllib.request.urlopen(req) as r:
+                try:
+                    parsed = json.load(r)
+                    log.debug(r.read().decode())
+                    return parsed
+                except json.JSONDecodeError:
+                    log.debug(r.read().decode())
+                    return False
+
+        except urllib.error.HTTPError:
             return False
 
     def send_message(self, channel, text):

tom/tag.py

@@ -1,6 +1,5 @@
 import re
 import os
-import requests
 import subprocess
 import datetime
 import hashlib

tom/utils.py

@@ -1,6 +1,8 @@
 import sys
 import json
 import hashlib
+import urllib.request
+import urllib.error
 
 
 def read_json(path):