feat: Update from origin

.gitattributes

@@ -0,0 +1,11 @@
+* text=auto eol=lf
+
+*.json text
+*.md text
+*.py text
+*.svg text
+*.txt text
+*.yml text
+*.yaml text
+
+*.png binary

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+      include: "scope"

.github/workflows/scripts/icon_path_validator.py

@@ -2,13 +2,19 @@
 import pathlib
 import string
 from typing import Final
+from xml.etree import ElementTree
 
 
 class InvalidStructureException(Exception):
     def __init__(self, msg: str):
         super().__init__(msg)
 
 
+class InvalidSVGException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
 class IconValidator:
     def __init__(self):
         self._SOURCE_DIR: Final[str] = "src"
@@ -23,6 +29,16 @@ def validate(self) -> None:
         for file in self._FULL_IMAGES_DIR.iterdir():
             if file.name not in json_icons:
                 raise InvalidStructureException(f"{file.name} must be used, {file} isn't used!")
+            if file.name.lower().endswith(".svg"):
+                self._validate_svg(file)
+
+    def _validate_svg(self, file: pathlib.Path) -> None:
+        try:
+            with file.open("r", encoding="utf8") as f:
+                content: str = f.read()
+            ElementTree.fromstring(content)
+        except ElementTree.ParseError as e:
+            raise InvalidSVGException(f"Invalid SVG '{file.name}': {e}")
 
     def get_json_icons(self) -> set[str]:
         letters: list[str] = list(string.ascii_lowercase)

.github/workflows/scripts/order_validator.py

@@ -0,0 +1,64 @@
+import json
+import pathlib
+from typing import Final, Any
+
+
+class DuplicateKeyException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class OrderingException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class InvalidStructureException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class OrderValidator:
+    def __init__(self):
+        self._SOURCE_DIR: Final[str] = "src"
+        self._TECH_DIR: Final[str] = "technologies"
+        self._FULL_TECH_DIR: Final[pathlib.Path] = pathlib.Path(self._SOURCE_DIR).joinpath(self._TECH_DIR)
+
+    def validate(self) -> None:
+        if not self._FULL_TECH_DIR.is_dir():
+            raise InvalidStructureException(f"{self._FULL_TECH_DIR} is not a valid directory")
+        for tech_file in sorted(self._FULL_TECH_DIR.iterdir()):
+            if not tech_file.name.endswith(".json"):
+                continue
+            self._check_ordering(tech_file)
+
+    def _check_ordering(self, tech_file: pathlib.Path) -> None:
+        with tech_file.open("r", encoding="utf8") as f:
+            data: dict = json.load(f, object_pairs_hook=self._duplicate_key_validator)
+        if not isinstance(data, dict):
+            raise InvalidStructureException(f"{tech_file.name} root must be an object")
+        keys: list[str] = list(data.keys())
+        sorted_keys: list[str] = sorted(keys, key=lambda x: x.lower())
+        if keys != sorted_keys:
+            mismatches: list[str] = [
+                f"Found '{actual}' expected '{expected}'"
+                for actual, expected in zip(keys, sorted_keys)
+                if actual != expected
+            ]
+            raise OrderingException(
+                f"{tech_file.name} entries are not ordered alphabetically by keys:\n  " +
+                "\n  ".join(mismatches)
+            )
+
+    @classmethod
+    def _duplicate_key_validator(cls, pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, value in pairs:
+            if key in result:
+                raise DuplicateKeyException(f"Duplicate key found: '{key}'")
+            result[key] = value
+        return result
+
+
+if __name__ == '__main__':
+    OrderValidator().validate()

.github/workflows/scripts/schema_validator.py

@@ -0,0 +1,38 @@
+import json
+import pathlib
+from typing import Final
+
+from jsonschema import validate, ValidationError
+
+
+class SchemaValidationException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class SchemaValidator:
+    def __init__(self):
+        self._SOURCE_DIR: Final[str] = "src"
+        self._TECH_DIR: Final[str] = "technologies"
+        self._FULL_TECH_DIR: Final[pathlib.Path] = pathlib.Path(self._SOURCE_DIR).joinpath(self._TECH_DIR)
+        self._SCHEMA_FILE: Final[pathlib.Path] = pathlib.Path("schema.json")
+
+    def validate(self) -> None:
+        if not self._SCHEMA_FILE.is_file():
+            raise FileNotFoundError(f"Schema file '{self._SCHEMA_FILE}' not found!")
+        with self._SCHEMA_FILE.open("r", encoding="utf8") as f:
+            schema: dict = json.load(f)
+        for tech_file in sorted(self._FULL_TECH_DIR.iterdir()):
+            if not tech_file.name.endswith(".json"):
+                continue
+            with tech_file.open("r", encoding="utf8") as f:
+                technologies: dict = json.load(f)
+            try:
+                validate(instance=technologies, schema=schema)
+            except ValidationError as e:
+                path: str = " -> ".join(str(p) for p in e.absolute_path) if e.absolute_path else "root"
+                raise SchemaValidationException(f"{tech_file.name}: {e.message} (at {path})")
+
+
+if __name__ == '__main__':
+    SchemaValidator().validate()

.github/workflows/scripts/technology_validator.py

@@ -74,6 +74,16 @@ def __init__(self, msg: str):
         super().__init__(msg)
 
 
+class TechNotFoundException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class InvalidURLException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
 class AbstractValidator:
     def __init__(self, required: bool = False):
         self._required = required
@@ -185,6 +195,27 @@ def get_type(self) -> list[Type]:
         return [str]
 
 
+class URLValidator(StringValidator):
+    def __init__(self, required: bool = False):
+        super().__init__(required)
+        self._url_pattern: Final[re.Pattern] = re.compile(
+            r"^https?://"
+            r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+"
+            r"[A-Za-z0-9-]{2,}"
+            r"(?::\d+)?"
+            r"(?:/[^\s]*)?"
+            r"$"
+        )
+
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        if not self._url_pattern.match(data):
+            self._set_custom_error(InvalidURLException(f"Tech '{tech_name}' has invalid URL: '{data}'"))
+            return False
+        return True
+
+
 class BoolValidator(AbstractValidator):
     def get_type(self) -> list[Type]:
         return [bool]
@@ -200,6 +231,26 @@ def get_type(self) -> list[Type]:
         return [dict]
 
 
+class DNSValidator(DictValidator):
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        for k, v in data.items():
+            if not isinstance(k, str):
+                self._set_custom_error(InvalidKeyException(f"key in DNS for tech '{tech_name}' has an invalid type. 'str' is required, got type '{type(k).__name__}' -> '{k}'"))
+                return False
+            if not isinstance(v, list):
+                self._set_custom_error(InvalidKeyException(f"value in DNS for tech '{tech_name}' has an invalid type. 'list' is required, got type '{type(v).__name__}' -> '{v}'"))
+                return False
+            for record in v:
+                if not isinstance(record, str):
+                    self._set_custom_error(InvalidTypeForFieldException(f"Invalid type for dns in tech '{tech_name}', selector '{v}' '{record}' key must be string!"))
+                    return False
+                if not self._validate_regex(tech_name, record):
+                    return False
+        return True
+
+
 class CategoryValidator(ArrayValidator):
     def __init__(self, categories: list[int], required: bool = False):
         super().__init__(required)
@@ -297,6 +348,22 @@ def _validate(self, tech_name: str, data: Any) -> bool:
         return True
 
 
+class ReferenceValidator(ArrayValidator):
+    def __init__(self, all_techs: set[str]):
+        super().__init__()
+        self._all_techs: Final[set[str]] = all_techs
+
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        for ref in data:
+            clean_ref: str = ref.split(r"\;")[0]
+            if clean_ref not in self._all_techs:
+                self._set_custom_error(TechNotFoundException(f"Tech '{tech_name}' references '{clean_ref}' but it doesn't exist!"))
+                return False
+        return True
+
+
 class TechnologiesValidator:
     def __init__(self, file_name: str):
         self._SOURCE_DIR: Final[str] = "src"
@@ -308,22 +375,23 @@ def __init__(self, file_name: str):
         self._IMAGES_DIR: Final[str] = "images"
         self._ICONS_DIR: Final[str] = "icons"
         self._ICONS: Final[list[str]] = [icon.name for icon in pathlib.Path(self._SOURCE_DIR).joinpath(self._IMAGES_DIR).joinpath(self._ICONS_DIR).iterdir()]
+        self._ALL_TECHS: Final[set[str]] = self._get_all_tech_names()
         self._validators: dict[str, AbstractValidator] = {  # TODO confidence and version validator
             "cats": CategoryValidator(self._CATEGORIES, True),
-            "website": StringValidator(True),
+            "website": URLValidator(True),
             "description": StringValidator(),
             "icon": IconValidator(self._ICONS),
             "cpe": CPEValidator(),
             "saas": BoolValidator(),
             "oss": BoolValidator(),
             "pricing": PricingValidator(),
-            "implies": ArrayValidator(),  # TODO cat validation
-            "requires": ArrayValidator(),  # TODO ^
-            "excludes": ArrayValidator(),  # TODO ^
+            "implies": ReferenceValidator(self._ALL_TECHS),
+            "requires": ReferenceValidator(self._ALL_TECHS),
+            "excludes": ReferenceValidator(self._ALL_TECHS),
             "requiresCategory": CategoryValidator(self._CATEGORIES),
             "cookies": DictValidator(contains_regex=True),
             "dom": DomValidator(),
-            "dns": DictValidator(contains_regex=True),
+            "dns": DNSValidator(contains_regex=True),
             "js": DictValidator(contains_regex=True),
             "headers": DictValidator(contains_regex=True),
             "text": ArrayValidator(contains_regex=True),
@@ -365,6 +433,16 @@ def _duplicate_key_validator(cls, pairs: list[tuple[str, Any]]) -> dict[str, Any
             result[key] = value
         return result
 
+    def _get_all_tech_names(self) -> set[str]:
+        all_techs: set[str] = set()
+        for letter in list(string.ascii_lowercase) + ["_"]:
+            tech_file: pathlib.Path = self._FULL_TECH_DIR.joinpath(f"{letter}.json")
+            if tech_file.exists():
+                with tech_file.open("r", encoding="utf8") as f:
+                    technologies: dict = json.load(f)
+                all_techs.update(technologies.keys())
+        return all_techs
+
 
 class TechnologyProcessor:
     def __init__(self, tech_name: str, tech_data: dict, validators: dict[str, AbstractValidator]):
@@ -389,4 +467,4 @@ def process(self) -> None:
 if __name__ == '__main__':
     # for letter in string.ascii_lowercase + "_":
     #     TechnologiesValidator(os.getenv("TECH_FILE_NAME", f"{letter}.json")).validate()
-    TechnologiesValidator(os.getenv("TECH_FILE_NAME", f"a.json")).validate()
+    TechnologiesValidator(os.getenv("TECH_FILE_NAME")).validate()

.github/workflows/tech_matrix_prep.yml

@@ -7,14 +7,14 @@ on:
         value: ${{ jobs.tech_prep.outputs.technologies }}
 jobs:
   tech_prep:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: get current technology matrix
         id: getTechMatrix
         run: python3 .github/workflows/scripts/tech_list_matrix_generator.py
 
     outputs:
-      technologies: ${{ steps.getTechMatrix.outputs.technologies }}
+      technologies: ${{ steps.getTechMatrix.outputs.technologies }}