Add some CI for security testing

.gitea/workflows/datadog-sca.yml

@@ -0,0 +1,18 @@
+on: [push]
+
+name: Datadog Software Composition Analysis
+
+jobs:
+  software-composition-analysis:
+    runs-on: ubuntu-latest
+    name: Datadog SBOM Generation and Upload
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Check imported libraries are secure and compliant
+      id: datadog-software-composition-analysis
+      uses: DataDog/datadog-sca-github-action@main
+      with:
+        dd_api_key: ${{ secrets.DD_API_KEY }}
+        dd_app_key: ${{ secrets.DD_APP_KEY }}
+        dd_site: datadoghq.com

.gitea/workflows/datadog-static-analysis.yml

@@ -0,0 +1,19 @@
+on: [push]
+
+name: Datadog Static Analysis
+
+jobs:
+  static-analysis:
+    runs-on: ubuntu-latest
+    name: Datadog Static Analyzer
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Check code meets quality and security standards
+      id: datadog-static-analysis
+      uses: DataDog/datadog-static-analyzer-github-action@v1
+      with:
+        dd_api_key: ${{ secrets.DD_API_KEY }}
+        dd_app_key: ${{ secrets.DD_APP_KEY }}
+        dd_site: datadoghq.com
+        cpu_count: 2

src/rendering/helpers.py

@@ -1,32 +1,39 @@
 from dataclasses import dataclass
 from src.config.config import Configuration
 from src.rendering import GENERIC_FILE_MAPPING
+from src.rendering.markdown import render_markdown
 from enum import Enum
 
 from thumbhash import image_to_thumbhash
 from PIL import Image
+from datetime import datetime
+import frontmatter
 
 
 @dataclass
 class ImageMetadata:
-    path: str
     width: int
     height: int
     alt: str
-    thumbhash: str
-    # exif attributes
     exif: dict
 
+@dataclass
+class MarkdownMetadata:
+    fontmatter: dict
+    content: str
+    preview: str
+
 
 @dataclass
 class FileMetadata:
-    path: str
+    typeMeta: MarkdownMetadata | None
-    first_hundred_chars: str
 
 
 @dataclass
 class TemplateFile:
     name: str
+    path: str
+    proper_name: str
     extension: str
     categories: list[str]
     date_modified: str
@@ -37,10 +44,47 @@ class TemplateFile:
     is_dir: bool
 
 
+def format_date(timestamp):
+    return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d")
+
+
 class TemplateHelpers:
     def __init__(self, config: Configuration):
         self.config: Configuration = config
 
+
+    def build_metadata_for_file(self, path: str, categories: list[str] = []):
+        """Builds metadata for a file"""
+        file_path = self.config.content_dir / path
+        for k in categories:
+            if k == "image":
+                img = Image.open(file_path)
+                exif = img._getexif()
+                orientation = exif.get(274, 1) if exif else 1
+                width, height = img.width, img.height
+                if orientation in [5, 6, 7, 8]:
+                    width, height = height, width
+                return ImageMetadata(
+                    width=width,
+                    height=height,
+                    alt=file_path.name,
+                    exif=img.info,
+                )
+            elif k == "document":
+                ret = None
+                with open(file_path, "r") as fdoc:
+                    ret = FileMetadata(None)
+                if file_path.suffix[1:].lower() == "md":
+                    ret.typeMeta = MarkdownMetadata({}, "", "")
+                    ret.typeMeta.fontmatter = frontmatter.load(file_path)
+                    ret.typeMeta.content = render_markdown(file_path)
+                    ret.typeMeta.preview = ret.typeMeta.content[:100]
+                    if "#" in ret.typeMeta.preview:
+                        ret.typeMeta.preview = ret.typeMeta.preview.split("#")[0]
+                return ret
+        return None
+
+
     def get_folder_contents(self, path: str = ""):
         """Returns the contents of a folder as a list of TemplateFile objects
 
@@ -52,45 +96,25 @@ class TemplateHelpers:
         for f in files:
             t = TemplateFile(
                 name=f.name,
+                path=str(f.relative_to(self.config.content_dir)),
+                proper_name=f.stem,
                 extension=f.suffix.lower(),
                 categories=[],
-                date_modified=f.stat().st_mtime,
+                date_modified=format_date(f.stat().st_mtime),
-                date_created=f.stat().st_ctime,
+                date_created=format_date(f.stat().st_ctime),
                 size_kb=f.stat().st_size / 1024,
                 metadata=None,
                 dir_item_count=len(list(f.glob("*"))) if f.is_dir() else 0,
                 is_dir=f.is_dir(),
             )
             if f.is_file():
-                # Build metadata depending on the mapping in GENERIC_FILE_MAPPING
                 for k, v in GENERIC_FILE_MAPPING.items():
                     if f.suffix[1:].lower() in v:
                         t.categories.append(k)
-                        if k == "image":
+            t.metadata = self.build_metadata_for_file(f, t.categories)
-                            img = Image.open(f)
-                            exif = img._getexif()
-                            orientation = exif.get(274, 1) if exif else 1
-                            width, height = img.width, img.height
-                            if orientation in [5, 6, 7, 8]:
-                                width, height = height, width
-                            t.metadata = ImageMetadata(
-                                path=str(f.relative_to(self.config.content_dir)),
-                                width=width,
-                                height=height,
-                                alt=f.name,
-                                thumbhash=image_to_thumbhash(img),
-                                exif=img.info,
-                            )
-                        elif k == "document":
-                            with open(f, "r") as fdoc:
-                                t.metadata = FileMetadata(
-                                    path=str(f.relative_to(self.config.content_dir)),
-                                    first_hundred_chars=fdoc.read(100),
-                                )
             ret.append(t)
         return ret
 
-
     def get_sibling_content_files(self, path: str = ""):
         search_contnet_path = self.config.content_dir / path
         files = search_contnet_path.glob("*")

src/rendering/image.py

@@ -43,4 +43,4 @@ def generate_thumbnail(image_path, resize_percent, min_width):
         img.save(thumbnail_io, format=img_format)
         thumbnail_io.seek(0)
 
-    return (thumbnail_io, img_format)
+    return (thumbnail_io.getvalue(), img_format)

src/routes/routes.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from src.config.config import Configuration
 from src.rendering.renderer import render_page, render_error_page
-from flask import send_file
+from flask import send_file 
 from src.rendering.image import generate_thumbnail
 from functools import lru_cache
 
@@ -10,7 +10,39 @@ class RouteManager:
     def __init__(self, config: Configuration):
         self.config = config
 
+    def _ensure_route(self, path: str):
+        """
+        Escapes the path for anything like
+         a path execution or injection attack
+         evaluates the path and ensures that it it does not
+         go above the self.content.content_dir
+        If any part of the path contains __, __{foldername}, or __{filename},
+         that is a hidden file or folder and should raise an exception
+        Any illegal path should raise an exception
+        """
+        file_path: Path = self.config.content_dir / (path if path else "index.md")
+        print(file_path)
+        if file_path < self.config.content_dir:
+            raise Exception("Illegal path")
+
+        for part in file_path.parts:
+            if part.startswith("__"):
+                raise Exception("Illegal path")
+
     def default_route(self, path: str):
+        try:
+            self._ensure_route(path)
+            print("all good")
+            print(path)
+            print("=============")
+        except Exception as e:
+            print(e)
+            return render_error_page(
+                403,
+                "Forbidden",
+                "You do not have permission to access this resource.",
+                self.config.templates_dir,
+            )
         file_path: Path = self.config.content_dir / (path if path else "index.md")
         return render_page(
             file_path,
@@ -18,26 +50,38 @@ class RouteManager:
             template_path=self.config.templates_dir,
             style_path=self.config.styles_dir,
         )
-        
+
     def get_style(self, path: str):
         file_path: Path = self.config.styles_dir / path
         if file_path.exists():
             return send_file(file_path)
         else:
-            return render_error_page(404, "Not Found", "The requested resource was not found on this server.", self.config.templates_dir)
+            return render_error_page(
+                404,
+                "Not Found",
+                "The requested resource was not found on this server.",
+                self.config.templates_dir,
+            )
 
-    @lru_cache(maxsize=128)
+    @lru_cache(maxsize=None)
     def get_static(self, path: str):
         file_path: Path = self.config.content_dir / path
         if file_path.exists():
-            # Check to see if the file is an image, if it is, render a thumbnail 
+            # Check to see if the file is an image, if it is, render a thumbnail
             if file_path.suffix.lower() in [".jpg", ".jpeg", ".png", ".gif"]:
-                thumbnail_bytes, img_format = generate_thumbnail(str(file_path), 10, 300)
+                thumbnail_bytes, img_format = generate_thumbnail(
+                    str(file_path), 10, 2048
+                )
                 return (
-                    thumbnail_bytes.getvalue(),
+                    thumbnail_bytes,
                     200,
                     {"Content-Type": f"image/{img_format.lower()}"},
                 )
             return send_file(file_path)
         else:
-            return render_error_page(404, "Not Found", "The requested resource was not found on this server.", self.config.templates_dir)
+            return render_error_page(
+                404,
+                "Not Found",
+                "The requested resource was not found on this server.",
+                self.config.templates_dir,
+            )