Utilities stress stability (#146)

* draft 1

* Update

* Fixing error related to multiple commissioning executions + enhancing logs

* total, readcommissionininfo and discovery plots

* added PASE plot and percentiles

* Update PoC with support to Utility screen.

* Adding support to analytics + enhancing container interface to support detach option.

* Added backend service to generate summary for logDisplay

* Removed unsued code

* Addes URL report in response for performance_summary endpoint

* Minor fixes

* Fixing util method for the repeat feature of the performance tests

* Merge alembic heads

* Cast project config

* Update method according to merged changes

* Make logs directory if it doesn't exist

* Add performance-logs to .gitignore

* Adding the Log Display web application scripts for install, uninstall, start and stop operations

* Adding support to Python Virtual Environment for the LogDisplay app scripts, plus minor changes

* Fixing the repeat test feature for regular tests with more than one iteration

* Removing configuration option for log folder in the LogDisplay tool for easier UX in the cost of less flexibility

* Force the creation of logs/ directory as well for the LogDisplay output folder

* Updating scripts and log generation to use environment variables for paths

* Changing the container's log output folder for path binded with host

* Updating the start script's path of the LogDisplay

* Disabling SIGINT trap from the start script so the python script inside receive the Ctrl+C interruption

* Adjustments on simulator script + moving matter_qa to outside endpoint + handling  stress script location

* Fixing lint issues.

* Fix lint

* fix additional lint issues

* lint

* fixing migration issues

* Removing unnecessary code.

* update log display feature.

* adjustments

---------

Co-authored-by: Fabio Maia <fabio_maia@apple.com>
Co-authored-by: Romulo Quidute Filho <rquidute@apple.com>
Co-authored-by: Antonio Melo Jr <a_junior@apple.com>
Co-authored-by: Carolina Lopes <ccruzagralopes@apple.com>

Author: 5 people

.flake8

@@ -8,3 +8,4 @@ per-file-ignores =
     test_collections/manual_tests/**/*:E501,W291
     test_collections/app1_tests/**/*:E501
     test_collections/semi_automated_tests/**/*:E501
+    alembic/versions/**/*:E128,W293,F401

.gitignore

@@ -12,4 +12,5 @@ test_environment.config
 SerialTests.lock
 test_db_creation.lock
 .sha_information
-test_collections/matter/sdk_tests/sdk_checkout
+test_collections/matter/sdk_tests/sdk_checkout
+performance-logs

.vscode/settings.json

@@ -12,7 +12,7 @@
         "editor.defaultFormatter": "ms-python.black-formatter", // black
         "editor.formatOnSave": true, // black
         "editor.codeActionsOnSave": {
-            "source.organizeImports": true // isort
+            "source.organizeImports": "explicit"
         },
     },
     // black

alembic/versions/0a251edfd975_adding_count_on_metadata_to_support_.py

@@ -0,0 +1,29 @@
+"""Adding count on metadata to support Performance Test
+
+Revision ID: 0a251edfd975
+Revises: 96ee37627a48
+Create Date: 2024-05-16 06:36:51.663230
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "0a251edfd975"
+down_revision = "e2c185af1226"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("testcasemetadata", sa.Column("count", sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("testcasemetadata", "count")
+    # ### end Alembic commands ###

alembic/versions/96ee37627a48_adding_the_new_column_collection_id.py

@@ -5,6 +5,7 @@
 Create Date: 2023-08-15 14:42:39.893126
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

alembic/versions/9df8004ad9bb_migrate_python_test_legacy_suite_.py

@@ -5,6 +5,7 @@
 Create Date: 2024-04-24 17:26:26.770729
 
 """
+
 from alembic import op
 
 

alembic/versions/e2c185af1226_pics_v2_support.py

@@ -5,6 +5,7 @@
 Create Date: 2024-06-19 11:46:15.158526
 
 """
+
 from alembic import op
 import sqlalchemy as sa
 

app/api/api_v1/endpoints/test_run_executions.py

@@ -14,9 +14,12 @@
 # limitations under the License.
 #
 import json
+import os
+from datetime import datetime
 from http import HTTPStatus
 from typing import Any, Dict, List, Optional
 
+import requests
 from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, UploadFile
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse, StreamingResponse
@@ -37,6 +40,10 @@
     selected_tests_from_execution,
 )
 from app.version import version_information
+from test_collections.matter.sdk_tests.support.performance_tests.utils import (
+    create_summary_report,
+)
+from test_collections.matter.test_environment_config import TestEnvironmentConfigMatter
 
 router = APIRouter()
 
@@ -479,3 +486,87 @@ def import_test_run_execution(
             status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
             detail=str(error),
         )
+
+
+date_pattern_out_file = "%Y_%m_%d_%H_%M_%S"
+
+
+@router.post("/{id}/performance_summary")
+def generate_summary_log(
+    *,
+    db: Session = Depends(get_db),
+    id: int,
+    project_id: int,
+) -> JSONResponse:
+    """
+    Imports a test run execution to the the given project_id.
+    """
+
+    project = crud.project.get(db=db, id=project_id)
+
+    if not project:
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_FOUND, detail="Project not found"
+        )
+
+    project_config = TestEnvironmentConfigMatter(**project.config)
+    matter_qa_url = None
+    LOGS_FOLDER = "/test_collections/logs"
+    HOST_BACKEND = os.getenv("BACKEND_FILEPATH_ON_HOST") or ""
+    HOST_OUT_FOLDER = HOST_BACKEND + LOGS_FOLDER
+
+    if (
+        project_config.test_parameters
+        and "matter_qa_url" in project_config.test_parameters
+    ):
+        matter_qa_url = project_config.test_parameters["matter_qa_url"]
+    else:
+        raise HTTPException(
+            status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+            detail="matter_qa_url must be configured",
+        )
+
+    page = requests.get(f"{matter_qa_url}/home")
+    if page.status_code is not int(HTTPStatus.OK):
+        raise HTTPException(
+            status_code=page.status_code,
+            detail=(
+                "The LogDisplay server is not responding.\n"
+                "Verify if the tool was installed, configured and initiated properly"
+            ),
+        )
+
+    commissioning_method = project_config.dut_config.pairing_mode
+
+    test_run_execution = crud.test_run_execution.get(db=db, id=id)
+    if not test_run_execution:
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_FOUND, detail="Test Run Execution not found"
+        )
+
+    log_lines_list = log_utils.convert_execution_log_to_list(
+        log=test_run_execution.log, json_entries=False
+    )
+
+    timestamp = ""
+    if test_run_execution.started_at:
+        timestamp = test_run_execution.started_at.strftime(date_pattern_out_file)
+    else:
+        timestamp = datetime.now().strftime(date_pattern_out_file)
+
+    tc_name, execution_time_folder = create_summary_report(
+        timestamp, log_lines_list, commissioning_method
+    )
+
+    target_dir = f"{HOST_OUT_FOLDER}/{execution_time_folder}/{tc_name}"
+    url_report = f"{matter_qa_url}/home/displayLogFolder?dir_path={target_dir}"
+
+    summary_report: dict = {}
+    summary_report["url"] = url_report
+
+    options: dict = {"media_type": "application/json"}
+
+    return JSONResponse(
+        jsonable_encoder(summary_report),
+        **options,
+    )

app/models/test_case_metadata.py

@@ -31,6 +31,8 @@ class TestCaseMetadata(Base):
     id: Mapped[int] = mapped_column(primary_key=True, index=True)
     public_id: Mapped[str] = mapped_column(nullable=False)
 
+    count: Mapped[str] = mapped_column(Text, nullable=True)
+
     title: Mapped[str] = mapped_column(nullable=False)
     description: Mapped[str] = mapped_column(Text, nullable=False)
     version: Mapped[str] = mapped_column(nullable=False)

app/test_engine/models/test_case.py

@@ -62,6 +62,7 @@ def __init__(self, test_case_execution: TestCaseExecution):
         self.create_test_steps()
         self.__state = TestStateEnum.PENDING
         self.errors: List[str] = []
+        self.analytics: dict[str, str] = {}  # Move to dictionary
 
     # Make pics a class method as they are mostly needed at class level.
     @classmethod