migrate to table based tinydb

NOTE to migrate database install tinydb & run migrate.py

- added new functions to filter data with db queries
- in the storage module

webapp/data/migrate.py

@@ -0,0 +1,59 @@
+import os, json
+from tinydb import TinyDB
+from tinydb.storages import Storage
+
+# Paths
+OLD_FILE = "orders.json"
+ARCHIVE_FILE = "orders_old.json"
+NEW_FILE = "orders_new.json"  # final TinyDB JSON file
+TABLE_NAME = "orders"
+
+
+class PrettyJSONStorage(Storage):
+    """Custom TinyDB storage that pretty-prints JSON."""
+    def __init__(self, path):
+        self.path = path
+
+    def read(self):
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except FileNotFoundError:
+            return None
+
+    def write(self, data):
+        with open(self.path, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def migrate():
+    if not os.path.isfile(OLD_FILE):
+        print("No old JSON file found. Nothing to migrate.")
+        return
+
+    # Load old orders
+    with open(OLD_FILE, "r", encoding="utf-8") as f:
+        orders = json.load(f) or []
+
+    if not orders:
+        print("No orders found in the old JSON.")
+        return
+    
+    print(orders)
+
+    # Open TinyDB and insert into table
+    db = TinyDB(NEW_FILE, storage=PrettyJSONStorage)
+    orders_table = db.table(TABLE_NAME)
+    for order in orders:
+        orders_table.insert(order)
+
+    # Rename old file to archive
+    os.rename(OLD_FILE, ARCHIVE_FILE)
+    os.rename(NEW_FILE, OLD_FILE)
+    print(f"Migrated {len(orders)} orders into table '{TABLE_NAME}'.")
+    print(f"Old JSON file renamed to '{ARCHIVE_FILE}'.")
+    print(f"New TinyDB JSON file is '{OLD_FILE}'.")
+
+
+if __name__ == "__main__":
+    migrate()

webapp/routes/archive.py

@@ -2,18 +2,14 @@
 from __future__ import annotations
 import time
 from flask import Blueprint, render_template, redirect, url_for
-from webapp.storage.orders import load_orders, update_order
+from webapp.storage.orders import load_archived, mark_archived
 
 bp = Blueprint("archive", __name__)
 
 @bp.get("/archiv")
 def archive_index():
     """Zeigt ausschließlich Aufträge mit status == 'archived'."""
-    archived = [o for o in load_orders() if o.get("status") == "archived"]
-    archived.sort(
-        key=lambda o: (o.get("archived_at") or o.get("done_at") or o.get("created_at") or ""),
-        reverse=True,
-    )
+    archived = load_archived()
     return render_template("archive.html", archived_orders=archived)
 
 @bp.post("/archiv/add/<order_id>")
@@ -23,5 +19,5 @@ def archive_add(order_id: str):
     Danach BLEIBEN wir auf der Strukturen-Seite (kein Wechsel zur Archiv-Seite).
     """
     now = time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime())
-    update_order(order_id, {"status": "archived", "archived_at": now})
+    mark_archived(order_id)
     return redirect(url_for("structures.structures"))  # zurück zu /strukturen

webapp/routes/cookbook.py

@@ -10,17 +10,17 @@ from webapp.services.ccookbook import (
 )
 # Storage
 from webapp.storage.orders import (
-    load_orders, update_order,
-    cache_costs, cache_materials,
+    update_order, get_order_by_id,
+    cache_costs, cache_materials, load_pending
 )
 
 bp = Blueprint("cookbook", __name__, url_prefix="/api")
 
+# ----------------------------- Helpers ----------------------------------------
+
 def _now() -> str:
     return time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime())
 
-# ----------------------------- Helpers ----------------------------------------
-
 def _arg(name: str, default: str = "") -> str:
     v = request.args.get(name)
     return v if v is not None else default
@@ -31,12 +31,6 @@ def _to_int(s: str, default: int = 0) -> int:
     except Exception:
         return default
 
-def _order_by_id(order_id: str) -> Dict[str, Any] | None:
-    for o in load_orders():
-        if o.get("id") == order_id:
-            return o
-    return None
-
 # ----------------------------- Endpoints – Raw --------------------------------
 
 @bp.get("/cookbook/build_cost")
@@ -113,7 +107,7 @@ def api_cache(order_id: str):
 
 @bp.post("/order/<order_id>/refresh")
 def api_refresh_one(order_id: str):
-    o = _order_by_id(order_id)
+    o = get_order_by_id(order_id)
     if not o:
         return jsonify({"error": "order not found"}), 404
     bp_id = o.get("blueprint_type_id")
@@ -172,7 +166,7 @@ def api_refresh_one(order_id: str):
 
 @bp.post("/orders/refresh_all")
 def api_refresh_all():
-    orders = [o for o in load_orders() if o.get("status") == "open" and o.get("blueprint_type_id")]
+    orders = [o for o in load_pending() if o.get("blueprint_type_id")]
     done = 0
     for o in orders:
         try:
@@ -195,8 +189,8 @@ def api_open_costs():
     """
     items: List[Dict[str, Any]] = []
     total = 0.0
-    for o in load_orders():
-        if o.get("status") != "open" or not o.get("blueprint_type_id"):
+    for o in load_pending():
+        if not o.get("blueprint_type_id"):
             continue
 
         entry: Dict[str, Any] = {
@@ -258,8 +252,8 @@ def api_open_materials():
         if cost is not None:
             row["cost"] += float(cost or 0)
 
-    for o in load_orders():
-        if o.get("status") != "open" or not o.get("blueprint_type_id"):
+    for o in load_pending():
+        if not o.get("blueprint_type_id"):
             continue
 
         mats = o.get("materials")

webapp/routes/structures.py

@@ -3,7 +3,7 @@ import uuid, time
 from typing import List, Dict, Any
 from flask import Blueprint, render_template, request, redirect, url_for
 
-from webapp.storage.orders import load_orders, add_order, mark_done
+from webapp.storage.orders import load_pending,load_done, add_order, mark_done
 from webapp.services.blueprints import resolve_blueprint_id
 
 bp = Blueprint("structures", __name__)
@@ -16,12 +16,7 @@ REAC_STRUCTS= ["Athanor", "Tatara"]
 REAC_RIGS   = ["None", "T1", "T2"]
 
 def _split_orders() -> Dict[str, List[Dict[str, Any]]]:
-    orders = load_orders()
-    open_orders = [o for o in orders if o.get("status") == "open"]
-    done_orders = [o for o in orders if o.get("status") == "done"]
-    open_orders.sort(key=lambda o: o.get("created_at",""), reverse=True)
-    done_orders.sort(key=lambda o: o.get("done_at","") or "", reverse=True)
-    return {"open": open_orders, "done": done_orders}
+    return {"open": load_pending(), "done": load_done()}
 
 @bp.route("/strukturen", methods=["GET", "POST"])
 def structures():

webapp/storage/orders.py

@@ -1,55 +1,117 @@
 from __future__ import annotations
 import os, json, time
 from typing import List, Dict, Any
-from flask import current_app
+from flask import current_app, g
+from tinydb import TinyDB, Query
+from tinydb.storages import Storage
 
 FILE_NAME = "orders.json"
 
-def _file_path() -> str:
-    return os.path.join(current_app.root_path, "data", FILE_NAME)
+PRODUCTION = 'dev'
+
+class PrettyJSONStorage(Storage):
+    """Custom TinyDB storage that pretty-prints JSON."""
+    def __init__(self, path):
+        self.path = path
+
+    def read(self):
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except FileNotFoundError:
+            return None
+
+    def write(self, data):
+        with open(self.path, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+
+def get_db() -> TinyDB:
+    """
+    Return a persistent TinyDB instance per Flask app context.
+    Stored in `g` to avoid reopening the file multiple times.
+    """
+    if "orders_db" not in g:
+        db_path = os.path.join(current_app.root_path, "data", FILE_NAME)
+        if PRODUCTION == "dev":
+            g.orders_db = TinyDB(db_path, storage=PrettyJSONStorage )
+        else:
+            g.orders_db = TinyDB(db_path)
+    return g.orders_db
+
+
+def get_orders_table():
+    """Return the orders table."""
+    return get_db().table("orders")
 
 def load_orders() -> List[Dict[str, Any]]:
-    p = _file_path()
-    if not os.path.isfile(p): return []
-    try:
-        with open(p, "r", encoding="utf-8") as f:
-            return json.load(f) or []
-    except Exception:
-        return []
+    """Return all orders as a list of dicts."""
+    table = get_orders_table()
+    return table.all()
 
-def _save_orders(orders: List[Dict[str, Any]]) -> None:
-    p = _file_path()
-    os.makedirs(os.path.dirname(p), exist_ok=True)
-    with open(p, "w", encoding="utf-8") as f:
-        json.dump(orders, f, ensure_ascii=False, indent=2)
+def get_order_by_id(order_id: str) -> Dict[str, Any] | None:
+    """Return a single order by its ID, or None if not found."""
+    table = get_orders_table()
+    Order = Query()
+    result = table.get(Order.id == order_id)
+    return result
+
+def load_archived() -> List[Dict[str, Any]]:
+    """Return all archived orders, sorted by archived/done/created date descending."""
+    table = get_orders_table()
+    Order = Query()
+    archived = table.search(Order.status == "archived")
+
+    def sort_key(o: Dict[str, Any]) -> str:
+        return o.get("archived_at") or o.get("done_at") or o.get("created_at") or ""
+
+    archived.sort(key=sort_key, reverse=True)
+    return archived
+
+def load_pending() -> List[Dict[str, Any]]:
+    """Return all orders with status 'open', sorted by created_at descending."""
+    table = get_orders_table()
+    Order = Query()
+    pending = table.search(Order.status == "open")
+    pending.sort(key=lambda o: o.get("created_at", ""), reverse=True)
+    return pending
+
+
+def load_done() -> List[Dict[str, Any]]:
+    """Return all orders with status 'done', sorted by done_at descending."""
+    table = get_orders_table()
+    Order = Query()
+    done = table.search(Order.status == "done")
+    done.sort(key=lambda o: o.get("done_at") or "", reverse=True)
+    return done
 
 def add_order(order: Dict[str, Any]) -> None:
-    orders = load_orders()
-    orders.append(order)
-    _save_orders(orders)
+    table = get_orders_table()
+    table.insert(order)
+
 
 def mark_done(order_id: str) -> bool:
-    orders = load_orders()
-    changed = False
+    table = get_orders_table()
+    Order = Query()
     now = time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime())
-    for o in orders:
-        if o.get("id") == order_id and o.get("status") != "done":
-            o["status"] = "done"
-            o["done_at"] = now
-            changed = True
-    if changed: _save_orders(orders)
-    return changed
+    updated = table.update({"status": "done", "done_at": now}, Order.id == order_id)
+    return bool(updated)
+
+def mark_archived(order_id: str) -> bool:
+    """Mark an order as archived and set archived_at timestamp."""
+    now = time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime())
+    table = get_orders_table()
+    Order = Query()
+    updated = table.update({"status": "archived", "archived_at": now}, Order.id == order_id)
+    return bool(updated)
 
 def update_order(order_id: str, patch: Dict[str, Any]) -> bool:
-    orders = load_orders()
-    ok = False
-    for o in orders:
-        if o.get("id") == order_id:
-            o.update(patch)
-            ok = True
-            break
-    if ok: _save_orders(orders)
-    return ok
+    table = get_orders_table()
+    Order = Query()
+    updated = table.update(patch, Order.id == order_id)
+    return bool(updated)
+
 
 def cache_costs(order_id: str, costs: Dict[str, Any]) -> bool:
     now = time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime())
@@ -57,4 +119,4 @@ def cache_costs(order_id: str, costs: Dict[str, Any]) -> bool:
 
 def cache_materials(order_id: str, mats: Dict[str, Any]) -> bool:
     now = time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime())
-    return update_order(order_id, {"materials": mats, "last_updated": now})
+    return update_order(order_id, {"materials": mats, "last_updated": now})