diff --git a/api_app/analyzers_manager/migrations/0190_remove_greynoise_labs_analyzer.py b/api_app/analyzers_manager/migrations/0190_remove_greynoise_labs_analyzer.py new file mode 100644 index 0000000000..97522d091b --- /dev/null +++ b/api_app/analyzers_manager/migrations/0190_remove_greynoise_labs_analyzer.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.27 on 2026-04-10 + +from django.db import migrations + + +def migrate(apps, schema_editor): + PythonModule = apps.get_model("api_app", "PythonModule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + pm = PythonModule.objects.filter( + module="greynoise_labs.GreynoiseLabs", + base_path="api_app.analyzers_manager.observable_analyzers", + ).first() + if pm: + task_ids = [getattr(pm, "update_task_id", None)] + task_ids.extend(getattr(c, "health_check_task_id", None) for c in pm.analyzerconfigs.all()) + PeriodicTask.objects.filter(id__in=[t for t in task_ids if t]).delete() + + pm.analyzerconfigs.all().delete() + pm.delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("api_app", "0062_alter_parameter_python_module"), + ("analyzers_manager", "0189_update_capa_timeout"), + ("django_celery_beat", "0001_initial"), + ] + + operations = [ + migrations.RunPython(migrate, migrations.RunPython.noop), + ] diff --git a/api_app/analyzers_manager/observable_analyzers/greynoise_labs.py b/api_app/analyzers_manager/observable_analyzers/greynoise_labs.py deleted file mode 100644 index bb850cec0c..0000000000 --- a/api_app/analyzers_manager/observable_analyzers/greynoise_labs.py +++ /dev/null @@ -1,130 +0,0 @@ -# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl -# See the file 'LICENSE' for copying permission. - -import logging -import os - -import requests -from django.conf import settings - -from api_app.analyzers_manager.classes import ObservableAnalyzer -from api_app.models import PluginConfig - -logger = logging.getLogger(__name__) - -url = "https://api.labs.greynoise.io/1/query" -db_name = "topc2s_ips.txt" -db_location = f"{settings.MEDIA_ROOT}/{db_name}" - -queries = { - "noiserank": { - "query_string": "query NoiseRank($ip: String) { noiseRank(ip: $ip) \ - { queryInfo { resultsAvailable resultsLimit } ips { ip noise_score \ - sensor_pervasiveness country_pervasiveness payload_diversity \ - port_diversity request_rate } } }", - "ip_required": True, - }, - "topknocks": { - "query_string": "query TopKnocks($ip: String) { topKnocks(ip: $ip) \ - { queryInfo { resultsAvailable resultsLimit } knock { last_crawled \ - last_seen source_ip knock_port title favicon_mmh3_32 \ - favicon_mmh3_128 jarm ips emails links tor_exit headers apps } } } ", - "ip_required": True, - }, - "topc2s": { - "query_string": "query TopC2s { topC2s { queryInfo \ - { resultsAvailable resultsLimit } c2s { source_ip c2_ips \ - c2_domains payload hits pervasiveness } } } ", - "ip_required": False, - "db_location": db_location, - }, -} - - -class GreynoiseLabs(ObservableAnalyzer): - _auth_token: str - - def run(self): - result = {} - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self._auth_token}", - } - - for key, value in queries.items(): - if not value["ip_required"]: - if not os.path.isfile(value["db_location"]) and not self.update(): - error_message = f"Failed extraction from {key} db" - self.report.errors.append(error_message) - self.report.save() - logger.error(error_message) - continue - - with open(value["db_location"], "r", encoding="utf-8") as f: - db = f.read() - - db_list = db.split("\n") - if self.observable_name in db_list: - result[key] = {"found": True} - else: - result[key] = {"found": False} - continue - - json_body = { - "query": value["query_string"], - "variables": {"ip": f"{self.observable_name}"}, - } - response = requests.post(headers=headers, json=json_body, url=url) - response.raise_for_status() - result[key] = response.json() - - return result - - @classmethod - def _get_auth_token(cls): - for plugin in PluginConfig.objects.filter( - parameter__python_module=cls.python_module, - parameter__is_secret=True, - parameter__name="auth_token", - ): - if plugin.value: - return plugin.value - return None - - @classmethod - def _update_db(cls, auth_token: str): - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {auth_token}", - } - - try: - logger.info("Fetching data from greynoise API (Greynoise_Labs).....") - response = requests.post( - headers=headers, - json={"query": queries["topc2s"]["query_string"]}, - url=url, - ) - response.raise_for_status() - topc2s_data = response.json() - - with open(db_location, "w", encoding="utf-8") as f: - for value in topc2s_data["data"]["topC2s"]["c2s"]: - ip = value["source_ip"] - if ip: - f.write(f"{ip}\n") - - if not os.path.exists(db_location): - return False - - logger.info("Data fetched from greynoise API (Greynoise_Labs).....") - return True - except Exception as e: - logger.exception(e) - - @classmethod - def update(cls): - auth_token = cls._get_auth_token() - if auth_token: - return cls._update_db(auth_token=auth_token) - return False diff --git a/api_app/analyzers_manager/observable_analyzers/hudsonrock.py b/api_app/analyzers_manager/observable_analyzers/hudsonrock.py index 789d8f5f21..cca6f695a8 100644 --- a/api_app/analyzers_manager/observable_analyzers/hudsonrock.py +++ b/api_app/analyzers_manager/observable_analyzers/hudsonrock.py @@ -103,6 +103,10 @@ def run(self): + self.get_param_url(["sortby", "page", "installed_software"]) ) response = requests.post(url, headers=headers, json={"login": self.observable_name}) + else: + raise AnalyzerConfigurationException( + f"Invalid GENERIC observable (not an email): {self.observable_name}" + " for HudsonRock" + ) else: raise AnalyzerConfigurationException( f"Invalid observable type {self.observable_classification}" diff --git a/api_app/analyzers_manager/observable_analyzers/validin.py b/api_app/analyzers_manager/observable_analyzers/validin.py index 9dacad4ae8..aed94b4baa 100644 --- a/api_app/analyzers_manager/observable_analyzers/validin.py +++ b/api_app/analyzers_manager/observable_analyzers/validin.py @@ -36,6 +36,7 @@ def _run_all_queries(self, endpoints, headers): logger.error(f"Query {query_name} failed") # we wont stop other quries from executing if one fails + continue final_response[f"{query_name}"] = response.json() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/connectors_manager/connectors/misp.py b/api_app/connectors_manager/connectors/misp.py index 36ed2a6184..eaf4214a98 100644 --- a/api_app/connectors_manager/connectors/misp.py +++ b/api_app/connectors_manager/connectors/misp.py @@ -118,11 +118,16 @@ def run(self): # append attribute name to event info event.info += f": {self._base_attr_obj.value}" - # add event to MISP Instance - misp_event = misp_instance.add_event(event, pythonify=True) - # add attributes to event on MISP Instance + # bulk: attach all attributes to the event object before sending for attr in attributes: - misp_instance.add_attribute(misp_event.id, attr) + event.add_attribute( + attr.type, + attr.value, + **{k: v for k, v in attr.to_dict().items() if k not in ("type", "value", "uuid")}, + ) + + # single request — event + all attributes sent together + misp_event = misp_instance.add_event(event, pythonify=True) return misp_instance.get_event(misp_event.id) diff --git a/frontend/src/constants/miscConst.js b/frontend/src/constants/miscConst.js index 8b84dbdc10..8d13c96518 100644 --- a/frontend/src/constants/miscConst.js +++ b/frontend/src/constants/miscConst.js @@ -28,9 +28,9 @@ export const localTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone; export const HistoryPages = Object.freeze({ JOB: "jobs", INVESTIGAITON: "investigations", - USER_EVENT: "user-events", - USER_DOMAIN_WILDCARD_EVENT: "user-domain-wildcard-events", - USER_IP_WILDCARD_EVENT: "user-ip-wildcard-events", + USER_EVENT: "user-evaluations", + USER_DOMAIN_WILDCARD_EVENT: "user-domain-wildcard-evaluations", + USER_IP_WILDCARD_EVENT: "user-ip-wildcard-evaluations", }); export const Classifications = Object.freeze({ diff --git a/integrations/malware_tools_analyzers/Dockerfile b/integrations/malware_tools_analyzers/Dockerfile index 7304bc2717..6948d5180b 100644 --- a/integrations/malware_tools_analyzers/Dockerfile +++ b/integrations/malware_tools_analyzers/Dockerfile @@ -60,10 +60,12 @@ RUN if [[ $TARGETARCH == "amd64" ]]; then \ # Build guelfo's PEFrame WORKDIR ${PROJECT_PATH}/peframe COPY requirements/peframe-requirements.txt ./ +# peframe-ds 6.1.0 uses array.tostring() which was removed in Python 3.9+, patch it after install RUN python3 -m venv venv \ && . venv/bin/activate \ && pip3 install --no-cache-dir --upgrade pip \ - && pip3 install --no-cache-dir -r peframe-requirements.txt --no-cache-dir + && pip3 install --no-cache-dir -r peframe-requirements.txt --no-cache-dir \ + && sed -i 's/\.tostring()/\.tobytes()/g' venv/lib/python3.*/site-packages/peframe/modules/features.py # Install guelfo's artifacts # there is no version management on this project so we just pull the most recent changes diff --git a/issues_latest.json b/issues_latest.json new file mode 100644 index 0000000000..d4fdb03484 --- /dev/null +++ b/issues_latest.json @@ -0,0 +1,41 @@ +[ + { + "number": 3653, + "title": "[BUG] job_pipeline task does not set job's final status on exception jobs get stuck in RUNNING forever", + "url": "https://github.com/intelowlproject/IntelOwl/issues/3653", + "labels": [ + "bug" + ] + }, + { + "number": 3647, + "title": "[Bug] HudsonRock analyzer crashes when GENERIC observable is not an email", + "url": "https://github.com/intelowlproject/IntelOwl/issues/3647", + "labels": [ + "bug" + ] + }, + { + "number": 3639, + "title": "Race condition in Job creation causes data loss", + "url": "https://github.com/intelowlproject/IntelOwl/issues/3639", + "labels": [ + "bug" + ] + }, + { + "number": 3623, + "title": "Cross-organization data leakage in plugin_state_viewer endpoint", + "url": "https://github.com/intelowlproject/IntelOwl/issues/3623", + "labels": [] + }, + { + "number": 3622, + "title": "History artifacts tabs are broken", + "url": "https://github.com/intelowlproject/IntelOwl/issues/3622", + "labels": [ + "bug", + "frontend" + ] + } +] \ No newline at end of file diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_greynoise_labs.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_greynoise_labs.py deleted file mode 100644 index ae88ade954..0000000000 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_greynoise_labs.py +++ /dev/null @@ -1,93 +0,0 @@ -from unittest.mock import patch - -from api_app.analyzers_manager.observable_analyzers.greynoise_labs import GreynoiseLabs -from tests.api_app.analyzers_manager.unit_tests.observable_analyzers.base_test_class import ( - BaseAnalyzerTest, -) -from tests.mock_utils import MockUpResponse - - -class GreynoiseLabsTestCase(BaseAnalyzerTest): - analyzer_class = GreynoiseLabs - - @classmethod - def get_extra_config(cls): - return {"_auth_token": "demo_token", "report": {"errors": []}} - - @staticmethod - def get_mocked_response(): - return patch( - "requests.post", - side_effect=[ - MockUpResponse( - { - "data": { - "noiseRank": { - "queryInfo": { - "resultsAvailable": 1, - "resultsLimit": 1, - }, - "ips": [ - { - "ip": "20.235.249.22", - "noise_score": 12, - "sensor_pervasiveness": "very low", - "country_pervasiveness": "low", - "payload_diversity": "very low", - "port_diversity": "very low", - "request_rate": "low", - } - ], - } - } - }, - 200, - ), - MockUpResponse( - { - "data": { - "topKnocks": { - "queryInfo": { - "resultsAvailable": 1, - "resultsLimit": 1, - }, - "knock": { - "last_crawled": "2024-01-01T00:00:00Z", - "last_seen": "2024-01-02T00:00:00Z", - "source_ip": "20.235.249.22", - "knock_port": 22, - }, - } - } - }, - 200, - ), - MockUpResponse( - { - "data": { - "topC2s": { - "queryInfo": { - "resultsAvailable": 3, - "resultsLimit": 3, - }, - "c2s": [ - { - "source_ip": "91.92.247.12", - "c2_ips": ["103.245.236.120"], - "c2_domains": [], - "hits": 11608, - }, - { - "source_ip": "14.225.208.190", - "c2_ips": ["14.225.213.142"], - "c2_domains": [], - "hits": 2091, - }, - ], - } - } - }, - 200, - ), - ], - ) diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_hudsonrock.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_hudsonrock.py index 9d9f8bf911..909f1816f3 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_hudsonrock.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_hudsonrock.py @@ -1,3 +1,4 @@ +from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException from api_app.analyzers_manager.observable_analyzers.hudsonrock import HudsonRock from tests.api_app.analyzers_manager.unit_tests.observable_analyzers.base_test_class import ( BaseAnalyzerTest, @@ -37,3 +38,11 @@ def get_mocked_response(): 200, ), ) + + def test_invalid_generic_raises_exception(self): + config = self.get_extra_config() + config["observable_name"] = "johndoe123" + + analyzer = self.analyzer_class(**config) + with self.assertRaises(AnalyzerConfigurationException): + analyzer.run() diff --git a/tests/api_app/connectors_manager/test_misp.py b/tests/api_app/connectors_manager/test_misp.py new file mode 100644 index 0000000000..ca5329a41d --- /dev/null +++ b/tests/api_app/connectors_manager/test_misp.py @@ -0,0 +1,145 @@ +from unittest.mock import MagicMock, patch + +from kombu import uuid + +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification +from api_app.connectors_manager.connectors.misp import MISP +from api_app.connectors_manager.models import ConnectorConfig, ConnectorReport +from api_app.models import Job, Parameter, PluginConfig +from tests import CustomTestCase + + +class MISPConnectorTestCase(CustomTestCase): + fixtures = [ + "api_app/fixtures/0001_user.json", + ] + + @staticmethod + def _get_misp_config(): + return ConnectorConfig.objects.get(name="MISP") + + @staticmethod + def _create_plugin_configs(config): + pcs = [] + for name in ("url_key_name", "api_key_name"): + param = Parameter.objects.get(python_module=config.python_module, name=name) + pc = PluginConfig.objects.create( + parameter=param, + value="https://misp.test" if "url" in name else "test-api-key", + for_organization=False, + owner=None, + connector_config=config, + ) + pcs.append(pc) + return pcs + + def _setup_job(self): + config = self._get_misp_config() + pcs = self._create_plugin_configs(config) + analyzable = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) + job = Job.objects.create( + analyzable=analyzable, + user=self.superuser, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, + ) + job.connectors_to_execute.set([config]) + return job, config, pcs + + @staticmethod + def _cleanup(job, config, pcs): + try: + ConnectorReport.objects.get(job=job, config=config).delete() + except ConnectorReport.DoesNotExist: + pass + analyzable = job.analyzable + job.delete() + analyzable.delete() + for pc in pcs: + pc.delete() + + @patch("api_app.connectors_manager.connectors.misp.MockPyMISP") + @patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") + def test_bulk_add_event_called_once(self, mock_pymisp_cls, mock_misp_cls): + """ + run() must call add_event exactly once with all attributes already + attached to the event object. add_attribute on the MISP instance + must never be called (that was the old N+1 pattern). + """ + mock_instance = MagicMock() + mock_pymisp_cls.return_value = mock_instance + mock_misp_cls.return_value = mock_instance + + mock_event = MagicMock() + mock_event.id = 42 + mock_instance.add_event.return_value = mock_event + mock_instance.get_event.return_value = {"Event": {"id": 42}} + + job, config, pcs = self._setup_job() + try: + connector = MISP(config) + connector.start(job.pk, {}, uuid()) + + report = ConnectorReport.objects.get(job=job, config=config) + self.assertEqual(report.status, ConnectorReport.STATUSES.SUCCESS) + + mock_instance.add_event.assert_called_once() + mock_instance.add_attribute.assert_not_called() + + event_arg = mock_instance.add_event.call_args[0][0] + self.assertGreaterEqual(len(event_arg.attributes), 1) + finally: + self._cleanup(job, config, pcs) + + @patch("api_app.connectors_manager.connectors.misp.MockPyMISP") + @patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") + def test_all_attributes_present_on_event(self, mock_pymisp_cls, mock_misp_cls): + """ + The event sent to MISP must contain the base attribute and the + link attribute — all in one shot. + """ + mock_instance = MagicMock() + mock_pymisp_cls.return_value = mock_instance + mock_misp_cls.return_value = mock_instance + + mock_event = MagicMock() + mock_event.id = 99 + mock_instance.add_event.return_value = mock_event + mock_instance.get_event.return_value = {"Event": {"id": 99}} + + job, config, pcs = self._setup_job() + try: + connector = MISP(config) + connector.start(job.pk, {}, uuid()) + + event_arg = mock_instance.add_event.call_args[0][0] + attr_types = [a.type for a in event_arg.attributes] + + self.assertIn("ip-src", attr_types) + self.assertIn("link", attr_types) + finally: + self._cleanup(job, config, pcs) + + @patch("api_app.connectors_manager.connectors.misp.MockPyMISP") + @patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") + def test_add_event_failure_marks_report_failed(self, mock_pymisp_cls, mock_misp_cls): + """ + If add_event raises, the connector report status must be FAILED. + """ + mock_instance = MagicMock() + mock_pymisp_cls.return_value = mock_instance + mock_misp_cls.return_value = mock_instance + mock_instance.add_event.side_effect = Exception("MISP unreachable") + + job, config, pcs = self._setup_job() + try: + connector = MISP(config) + try: + connector.start(job.pk, {}, uuid()) + except Exception: + pass + + report = ConnectorReport.objects.get(job=job, config=config) + self.assertEqual(report.status, ConnectorReport.STATUSES.FAILED) + finally: + self._cleanup(job, config, pcs) diff --git a/tests/test_crons.py b/tests/test_crons.py index 101daea696..0e3c90f77c 100644 --- a/tests/test_crons.py +++ b/tests/test_crons.py @@ -7,10 +7,8 @@ from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.file_analyzers import quark_engine, yara_scan -from api_app.analyzers_manager.models import AnalyzerConfig from api_app.analyzers_manager.observable_analyzers import ( feodo_tracker, - greynoise_labs, ja4_db, maxmind, phishing_army, @@ -19,8 +17,8 @@ tor_nodes_danmeuk, tweetfeeds, ) -from api_app.choices import Classification, PythonModuleBasePaths -from api_app.models import Job, Parameter, PluginConfig, PythonModule +from api_app.choices import Classification +from api_app.models import Job from intel_owl.tasks import check_stuck_analysis, remove_old_jobs from . import CustomTestCase, get_logger @@ -306,57 +304,3 @@ def create_yara_file(path): mock_zipfile.return_value.extractall.side_effect = create_yara_file result = yara_scan.YaraScan.update() self.assertTrue(result) - - @if_mock_connections( - patch( - "requests.post", - return_value=MockUpResponse( - { - "data": { - "topC2s": { - "queryInfo": { - "resultsAvailable": 1914, - "resultsLimit": 191, - }, - "c2s": [ - { - "source_ip": "91.92.247.12", - "c2_ips": ["103.245.236.120"], - "c2_domains": [], - "hits": 11608, - }, - { - "source_ip": "14.225.208.190", - "c2_ips": ["14.225.213.142"], - "c2_domains": [], - "hits": 2091, - "pervasiveness": 26, - }, - { - "source_ip": "157.10.53.101", - "c2_ips": ["14.225.208.190"], - "c2_domains": [], - "hits": 1193, - "pervasiveness": 23, - }, - ], - }, - }, - }, - 200, - ), - ) - ) - def test_greynoise_labs_updater(self, mock_post=None): - python_module = PythonModule.objects.get( - base_path=PythonModuleBasePaths.ObservableAnalyzer.value, - module="greynoise_labs.GreynoiseLabs", - ) - PluginConfig.objects.create( - value="test", - parameter=Parameter.objects.get(python_module=python_module, is_secret=True, name="auth_token"), - for_organization=False, - owner=None, - analyzer_config=AnalyzerConfig.objects.filter(python_module=python_module).first(), - ) - self.assertTrue(greynoise_labs.GreynoiseLabs.update())