3 http client

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

dpytools/__init__.py

@@ -1,5 +0,0 @@
-from http_clients.http import HttpClient
-from  config.config import Config
-from logger.logger import logger
-from slack.slack import SlackNotifier
-from sns.sns import Subscription, publish

dpytools/http_clients/http_custom.py

@@ -0,0 +1,69 @@
+import backoff
+import http.client
+from http.client import HTTPException
+from urllib.parse import urlparse
+import logging
+
+
+# Function to log retry attempts
+def log_retry(details):
+    logging.error(f"Request failed, retrying... Attempt #{details['tries']}")
+
+
+class HttpClient:
+    # Initialize HttpClient with a backoff_max value
+    def __init__(self, backoff_max=30):
+        self.backoff_max = backoff_max
+
+    # GET request method with exponential backoff
+    @backoff.on_exception(
+        backoff.expo, 
+        HTTPException,
+        max_time=30,  
+        on_backoff=log_retry
+    )
+    def get(self, url, *args, **kwargs):
+        timeout = kwargs.pop('timeout', None)
+        logging.info(f"Sending GET request to {url}")
+        return self._request("GET", url, timeout=timeout, *args, **kwargs)
+
+    # POST request method with exponential backoff
+    @backoff.on_exception(
+        backoff.expo, 
+        HTTPException,
+        max_time=30, 
+        on_backoff=log_retry,
+    )
+    def post(self, url, *args, **kwargs):
+        timeout = kwargs.pop('timeout', None)
+        logging.info(f"Sending POST request to {url}")
+        return self._request("POST", url, timeout=timeout, *args, **kwargs)
+
+    # Private method to handle the request
+    def _request(self, method, url, *args, **kwargs):
+        try:
+            headers = kwargs.pop('headers', {})
+
+            url_parts = urlparse(url)
+            https_connection = http.client.HTTPSConnection(url_parts.netloc)
+            path = url_parts.path or '/'
+            https_connection.request(method, path, headers=headers, *args, **kwargs)
+
+            response = https_connection.getresponse()
+            response_content = response.read()
+            https_connection.close() 
+
+            # Raise an exception if the HTTP status is 400 or above
+            if response.status >= 400:
+                raise HTTPException(f"HTTP request failed with status {response.status}, response: {response_content.decode()}")
+
+            return response
+
+        # Handle HTTPException separately to log and retry
+        except HTTPException as e:
+            logging.error(f"Request failed due to {str(e)}, retrying...")
+            raise
+        # Handle other exceptions
+        except Exception as e:
+            logging.error(f"An unexpected error occurred: {str(e)}")
+            raise

pyproject.toml

@@ -8,6 +8,9 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.9"
+requests = "^2.31.0"
+pytest = "^7.4.4"
+backoff = "^2.2.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.3"

tests/test_http.py

@@ -0,0 +1,114 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from http.client import HTTPResponse, HTTPException
+from dpytools.http_clients.http_custom import HttpClient  
+
+# Mock the HTTPSConnection class
+@patch('http.client.HTTPSConnection')
+def test_get(mock_connection):
+    """
+    Test that the get method returns a response object
+    """
+
+    # Create a mock response object
+    mock_response = MagicMock(HTTPResponse)
+    mock_response.status = 200
+    mock_response.read.return_value = b'Test response content'
+    mock_connection.return_value.getresponse.return_value = mock_response
+
+    # Create an instance of HttpClient and make a GET request
+    client = HttpClient()
+    response = client.get('http://example.com')
+
+    # Assertions to check the response status, content and the connection call
+    assert response.status == 200
+    assert response.read().decode() == 'Test response content'
+    mock_connection.assert_called_once_with('example.com')
+
+
+@patch('http.client.HTTPSConnection')
+def test_post(mock_connection):
+    """
+    Test that the post method returns a response object
+    """
+
+    # Create a mock response object
+    mock_response = MagicMock(HTTPResponse)
+    mock_response.status = 200
+    mock_response.read.return_value = b'Test response content'
+    mock_connection.return_value.getresponse.return_value = mock_response
+
+    # Create an instance of HttpClient and make a POST request
+    client = HttpClient()
+    response = client.post('http://example.com')
+
+    # Assertions to check the response status, content and the connection call
+    assert response.status == 200
+    assert response.read().decode() == 'Test response content'
+    mock_connection.assert_called_once_with('example.com')
+
+
+@patch('http.client.HTTPSConnection')
+def test_backoff_on_exception(mock_connection):
+    """
+    Test that the get method retries on HTTPException
+    """
+
+    # Create a mock response object
+    mock_response = MagicMock(HTTPResponse)
+    mock_response.status = 200
+
+    # Raise HTTPException on the first call, then return the mock_response
+    mock_connection.return_value.getresponse.side_effect = [HTTPException('HTTP Error'), mock_response]
+
+    # Create an instance of HttpClient and make a GET request
+    client = HttpClient()
+    response = client.get('http://example.com')
+
+    # Assertions to check the response status and the number of getresponse calls
+    assert response.status == 200
+    assert mock_connection.return_value.getresponse.call_count == 2
+
+
+@patch('http.client.HTTPSConnection')
+def test_request(mock_connection):
+    """
+    Test that the _request method returns a response object
+    """
+
+    # Create a mock response object
+    mock_response = MagicMock(HTTPResponse)
+    mock_response.status = 200
+    mock_response.read.return_value = b'Test response content'
+    mock_connection.return_value.getresponse.return_value = mock_response
+
+    # Create an instance of HttpClient and make a request
+    client = HttpClient()
+    response = client._request('GET', 'http://example.com')
+
+    # Assertions to check the response status, content and the connection call
+    assert response.status == 200
+    assert response.read().decode() == 'Test response content'
+    mock_connection.assert_called_once_with('example.com')
+
+
+@patch('http.client.HTTPSConnection')
+def test_request_with_timeout(mock_connection):
+    """
+    Test _request method with timeout passed as kwargs
+    """
+
+    # Create a mock response object
+    mock_response = MagicMock(HTTPResponse)
+    mock_response.status = 200
+    mock_response.read.return_value = b'Test response content'
+    mock_connection.return_value.getresponse.return_value = mock_response
+
+    # Create an instance of HttpClient and make a request with a timeout
+    client = HttpClient()
+    response = client._request('GET', 'http://example.com', timeout=5)
+
+    # Assertions to check the response status, content and the connection call
+    assert response.status == 200
+    assert response.read().decode() == 'Test response content'
+    mock_connection.assert_called_once_with('example.com')