Enable windows tests as github action (#626)

* align path access for OS agnostic support

* rely on os.sep when call in `split()` on path string
* lean in on Path `/` operator when combining with a Path object

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* retrieve executable from runtime

* avoid hard coded executbale path by asking runtime path
* guard file removal during cleanup

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* force report digest output as `utf-8`

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* test updates for windows temp files

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* set encoding for all file operations without `binary` flag

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* portable temp file remove for 3.10+

Remove temp files manually for OS portablity in python 3.10+
`delete_on_close` for temp files is added in [3.12](python/cpython#97015)

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* add windows pytest

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* update ggml temp file

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* variable name precision

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

* manual windows testing only

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

---------

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

Author: jmartin-tech

.github/workflows/test_windows.yml

@@ -0,0 +1,40 @@
+name: Garak pytest - Windows
+
+on: [workflow_dispatch]
+
+jobs:
+  build_windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: ["3.10","3.12"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: garak
+
+      - name: Checkout ecoji for modified windows install
+        uses: actions/checkout@v3
+        with:
+          repository: mecforlove/ecoji-py
+          path: ecoji-py
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd ecoji-py
+          echo "mitigate" > README.md
+          pip install setuptools
+          python setup.py install
+          cd ../garak
+          pip install -r requirements.txt
+
+      - name: Test with pytest
+        run: |
+          cd garak
+          python -m pytest tests/

garak/_config.py

@@ -114,7 +114,7 @@ def _store_config(settings_files) -> None:
 
 def load_base_config() -> None:
     global loaded
-    settings_files = [str(transient.basedir / "resources/garak.core.yaml")]
+    settings_files = [str(transient.basedir / "resources" / "garak.core.yaml")]
     logging.debug("Loading configs from: %s", ",".join(settings_files))
     _store_config(settings_files=settings_files)
     loaded = True
@@ -127,7 +127,7 @@ def load_config(
     # and then not have cli be upset when these are not given as cli params
     global loaded
 
-    settings_files = [str(transient.basedir / "resources/garak.core.yaml")]
+    settings_files = [str(transient.basedir / "resources" / "garak.core.yaml")]
 
     fq_site_config_filename = str(transient.basedir / site_config_filename)
     if os.path.isfile(fq_site_config_filename):

garak/analyze/misp.py

@@ -10,9 +10,13 @@
 
 from garak import _plugins
 
+# does this utility really have access to _config?
+misp_resource_file = (
+    garak._config.transient.basedir / "garak" / "resources" / "misp_descriptions.tsv"
+)
 misp_descriptions = {}
-if os.path.isfile("garak/resources/misp_descriptions.tsv"):
-    with open("garak/resources/misp_descriptions.tsv", "r", encoding="utf-8") as f:
+if os.path.isfile(misp_resource_file):
+    with open(misp_resource_file, "r", encoding="utf-8") as f:
         for line in f:
             key, title, descr = line.strip().split("\t")
             misp_descriptions[key] = (title, descr)

garak/analyze/report_avid.py

@@ -32,7 +32,7 @@
 # load up a .jsonl output file, take in eval and config rows
 report_location = _config.args.report
 print(f"📜 Converting garak reports {report_location}")
-with open(report_location, "r") as reportfile:
+with open(report_location, "r", encoding="utf-8") as reportfile:
     for line in reportfile:
         record = json.loads(line.strip())
         if record["entry_type"] == "eval":
@@ -114,6 +114,6 @@
 
 # save final output
 write_location = report_location.replace(".report", ".avid")
-with open(write_location, "w") as f:
+with open(write_location, "w", encoding="utf-8") as f:
     f.writelines(r.json() + "\n" for r in all_reports)
 print(f"📜 AVID reports generated at {write_location}")

garak/analyze/report_digest.py

@@ -16,7 +16,7 @@
 from garak import _config
 
 templateLoader = jinja2.FileSystemLoader(
-    searchpath=_config.transient.basedir / "analyze/templates/"
+    searchpath=_config.transient.basedir / "analyze" / "templates"
 )
 templateEnv = jinja2.Environment(loader=templateLoader)
 
@@ -28,13 +28,12 @@
 end_module = templateEnv.get_template("end_module.jinja")
 
 
+misp_resource_file = (
+    _config.transient.basedir / "garak" / "resources" / "misp_descriptions.tsv"
+)
 misp_descriptions = {}
-if os.path.isfile(_config.transient.basedir / "garak/resources/misp_descriptions.tsv"):
-    with open(
-        _config.transient.basedir / "garak/resources/misp_descriptions.tsv",
-        "r",
-        encoding="utf-8",
-    ) as f:
+if os.path.isfile(misp_resource_file):
+    with open(misp_resource_file, "r", encoding="utf-8") as f:
         for line in f:
             key, title, descr = line.strip().split("\t")
             misp_descriptions[key] = (title, descr)
@@ -70,7 +69,7 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy):
 
     digest_content = header_template.render(
         {
-            "reportfile": report_path.split("/")[-1],
+            "reportfile": report_path.split(os.sep)[-1],
             "garak_version": garak_version,
             "start_time": start_time,
             "run_uuid": run_uuid,
@@ -223,4 +222,4 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy):
     if len(sys.argv) == 3:
         taxonomy = sys.argv[2]
     digest_content = compile_digest(report_path, taxonomy=taxonomy)
-    print(digest_content)
+    print(digest_content.encode("utf-8"))

garak/detectors/riskywords.py

@@ -10,7 +10,9 @@
 from garak.detectors.base import StringDetector
 
 surge_list = defaultdict(list)
-with open(_config.transient.basedir / "resources/profanity_en.csv", "r") as csvfile:
+with open(
+    _config.transient.basedir / "resources" / "profanity_en.csv", "r", encoding="utf-8"
+) as csvfile:
     reader = csv.DictReader(csvfile)
     for row in reader:
         surge_list[row["category_1"]].append(row["text"])
@@ -22,7 +24,9 @@
 min_ofcom_strength = 2
 ofcom_list = defaultdict(list)
 with open(
-    _config.transient.basedir / "resources/ofcom-potentially-offensive.txt", "r"
+    _config.transient.basedir / "resources" / "ofcom-potentially-offensive.txt",
+    "r",
+    encoding="utf-8",
 ) as csvfile:
     reader = csv.DictReader(csvfile, delimiter="\t")
     for row in reader:
@@ -279,7 +283,9 @@ class LDNOOBW(StringDetector):
     def __init__(self):
         substrings = []
         with open(
-            _config.transient.basedir / "resources/ldnoobw-en.txt", "r"
+            _config.transient.basedir / "resources" / "ldnoobw-en.txt",
+            "r",
+            encoding="utf-8",
         ) as wordlist:
             for line in wordlist:
                 if line.strip():

garak/detectors/specialwords.py

@@ -4,7 +4,6 @@
 Matches terms in word lists
 """
 
-
 import json
 
 from garak import _config
@@ -26,7 +25,7 @@ class SlursReclaimedSlurs(StringDetector):
     def __init__(self):
         self.substrings = set([])
         with open(
-            _config.transient.basedir / "resources/slurprompts.jsonl",
+            _config.transient.basedir / "resources" / "slurprompts.jsonl",
             "r",
             encoding="utf-8",
         ) as slurfile:

garak/probes/continuation.py

@@ -43,7 +43,7 @@ def __init__(self):
         # append these to each of the prefixes for that term
         slur_prefixes = defaultdict(set)
         with open(
-            _config.transient.basedir / "resources/slurprompts_80.jsonl",
+            _config.transient.basedir / "resources" / "slurprompts_80.jsonl",
             "r",
             encoding="utf-8",
         ) as slurfile:

garak/probes/dan.py

@@ -536,10 +536,11 @@ class AutoDANCached(Probe):
 
     def __init__(
         self,
-        autodan_prompt_location: str = str(
-            garak._config.transient.basedir
-            / "resources/autodan/data/autodan_prompts.txt"
-        ),
+        autodan_prompt_location: str = garak._config.transient.basedir
+        / "resources"
+        / "autodan"
+        / "data"
+        / "autodan_prompts.txt",
     ):
         self.autodan_prompt_location = autodan_prompt_location
 
@@ -668,7 +669,9 @@ def __init__(self):
         super().__init__()
 
         inthewild_path = str(
-            garak._config.transient.basedir / "resources/inthewild_jailbreak_llms.txt"
+            garak._config.transient.basedir
+            / "resources"
+            / "inthewild_jailbreak_llms.txt"
         )
 
         with open(inthewild_path, "r", encoding="utf-8") as f:

garak/probes/encoding.py

@@ -7,7 +7,6 @@
 optional variable: "encoding.payloads" -> a list of payload types to be used
 """
 
-
 import base64
 import binascii
 import json
@@ -58,7 +57,7 @@ def load_payloads():
 
         payload_library["slur_terms"] = []
         with open(
-            _config.transient.basedir / "resources/slurprompts_80.jsonl",
+            _config.transient.basedir / "resources" / "slurprompts_80.jsonl",
             "r",
             encoding="utf-8",
         ) as slurfile: