diff --git a/backend/gds_builder.py b/backend/gds_builder.py index b52c688..26b0375 100644 --- a/backend/gds_builder.py +++ b/backend/gds_builder.py @@ -4,12 +4,11 @@ # Developer : Qin Yue @ 2026 # Organization : OptiHK Limited # ----------------------------------------------------------------------------- -import os +import tempfile from dataclasses import dataclass, field -from typing import Dict, List - -import yaml +from typing import List +from layout_files import load_layout_cell_files, write_layout_cells_to_directory from router_dependency import require_router_stack @@ -31,16 +30,25 @@ def build_project_gds( prefer_full_gds: bool = False, ) -> BuildResult: """Build a hierarchical project GDS from saved cell YAML files with mxpic_router.""" - cells = _load_project_cells(project_dir) + cells, warnings = _load_project_cells(project_dir) if not cells: raise ValueError("No saved cell YAML files found for this project") - return _build_with_mxpic_router( - project_dir, - output_path, - pdk_public_root, - technology_manifest_path, - prefer_full_gds, + with tempfile.TemporaryDirectory(prefix="mxpic_gds_project_") as staged_project_dir: + write_layout_cells_to_directory(cells, staged_project_dir) + result = _build_with_mxpic_router( + staged_project_dir, + output_path, + pdk_public_root, + technology_manifest_path, + prefer_full_gds, + ) + + return BuildResult( + output_path=result.output_path, + engine=result.engine, + cells_built=result.cells_built, + warnings=warnings + result.warnings, ) @@ -70,15 +78,6 @@ def _build_with_mxpic_router( ) -def _load_project_cells(project_dir: str) -> Dict[str, dict]: +def _load_project_cells(project_dir: str): """Load saved cell YAML documents from a project directory.""" - cells = {} - for filename in sorted(os.listdir(project_dir)): - if not filename.lower().endswith((".yml", ".yaml")): - continue - path = os.path.join(project_dir, filename) - with open(path, "r", encoding="utf-8") as file: - data = yaml.safe_load(file) or {} - cell_name = str(data.get("name") or os.path.splitext(filename)[0]) - cells[cell_name] = data - return cells + return load_layout_cell_files(project_dir) diff --git a/backend/layout_files.py b/backend/layout_files.py new file mode 100644 index 0000000..c5e52db --- /dev/null +++ b/backend/layout_files.py @@ -0,0 +1,126 @@ +# ----------------------------------------------------------------------------- +# Description: Layout YAML file filtering, parsing, and staging helpers. +# Inside functions: is_layout_cell_filename, parse_layout_cell_content, read_layout_cell_file, load_layout_cell_files, layout_cell_filename, write_layout_cells_to_directory +# Developer : Qin Yue @ 2026 +# Organization : OptiHK Limited +# ----------------------------------------------------------------------------- +import os +import re + +import yaml + + +LAYOUT_YAML_EXTENSIONS = (".yml", ".yaml") +ROUTE_SIDECAR_SUFFIXES = (".routes.yml", ".routes.yaml") + + +class LayoutFileError(ValueError): + """Raised when a saved layout YAML file cannot be used as a cell.""" + + +def is_layout_cell_filename(filename): + """Return True for user layout cell YAML files, excluding sidecars/manifests.""" + lower = (filename or "").lower() + if lower == "technology.yml": + return False + if lower.endswith(ROUTE_SIDECAR_SUFFIXES): + return False + return lower.endswith(LAYOUT_YAML_EXTENSIONS) + + +def is_layout_cell_document(data): + """Return True when parsed YAML looks like an mxPIC saved cell/project.""" + if not isinstance(data, dict): + return False + + kind = data.get("kind") + if kind and str(kind).strip().lower() != "cell": + return False + + layout_keys = { + "schema_version", + "canvas_size", + "canvasSize", + "instances", + "elements", + "pins", + "ports", + "bundles", + } + return any(key in data for key in layout_keys) + + +def parse_layout_cell_content(content, source="layout YAML"): + """Parse and validate saved layout YAML content.""" + try: + data = yaml.safe_load(content or "") or {} + except yaml.YAMLError as exc: + raise LayoutFileError(f"{source} is not valid YAML: {exc}") from exc + + if not is_layout_cell_document(data): + raise LayoutFileError(f"{source} is not a saved mxPIC layout cell") + return data + + +def read_layout_cell_file(path): + """Read a saved layout cell file and return parsed data plus raw content.""" + try: + with open(path, "r", encoding="utf-8") as file: + content = file.read() + except OSError as exc: + raise LayoutFileError(f"{os.path.basename(path)} could not be read: {exc}") from exc + return parse_layout_cell_content(content, os.path.basename(path)), content + + +def load_layout_cell_files(project_dir): + """Load valid layout cell files from a project directory and collect warnings.""" + cells = [] + warnings = [] + if not os.path.isdir(project_dir): + return cells, warnings + + for filename in sorted(os.listdir(project_dir)): + if not is_layout_cell_filename(filename): + continue + path = os.path.join(project_dir, filename) + if not os.path.isfile(path): + continue + try: + data, content = read_layout_cell_file(path) + except LayoutFileError as exc: + warnings.append(f"Skipped {filename}: {exc}") + continue + + cells.append({ + "filename": filename, + "name": str(data.get("name") or os.path.splitext(filename)[0]), + "data": data, + "content": content, + }) + return cells, warnings + + +def layout_cell_filename(cell_name, fallback="canvas_1.yml"): + """Build a safe filename for staging a parsed layout cell.""" + name = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(cell_name or "")).strip("._") + if not name: + return fallback + return f"{name}.yml" + + +def write_layout_cells_to_directory(cells, output_dir): + """Write valid layout cell contents into a clean staging directory.""" + os.makedirs(output_dir, exist_ok=True) + used_names = set() + for index, cell in enumerate(cells, start=1): + filename = layout_cell_filename(cell.get("name"), cell.get("filename") or f"cell_{index}.yml") + base, ext = os.path.splitext(filename) + unique_filename = filename + counter = 1 + while unique_filename.lower() in used_names: + counter += 1 + unique_filename = f"{base}_{counter}{ext}" + used_names.add(unique_filename.lower()) + + with open(os.path.join(output_dir, unique_filename), "w", encoding="utf-8") as file: + file.write(cell["content"]) diff --git a/backend/routed_layout_preview.py b/backend/routed_layout_preview.py index 305ffc5..cc0c411 100644 --- a/backend/routed_layout_preview.py +++ b/backend/routed_layout_preview.py @@ -9,6 +9,7 @@ import tempfile import yaml +from layout_files import load_layout_cell_files, write_layout_cells_to_directory from router_dependency import require_router_stack @@ -32,9 +33,20 @@ def create_routed_layout_svg( # Build into a temporary GDS first, then convert the generated top cell into # the SVG preview consumed by the canvas. with tempfile.TemporaryDirectory(prefix="mxpic_routed_preview_") as temp_dir: + staged_project_dir = os.path.join(temp_dir, "project") + saved_cells, _warnings = load_layout_cell_files(project_dir) + staged_cells = [cell for cell in saved_cells if cell["name"] != cell_name] + staged_cells.append({ + "filename": f"{cell_name}.yml", + "name": cell_name, + "data": layout, + "content": yaml_content, + }) + write_layout_cells_to_directory(staged_cells, staged_project_dir) + temp_gds = os.path.join(temp_dir, f"{cell_name}.gds") build_project_gds( - project_dir=project_dir, + project_dir=staged_project_dir, output_path=temp_gds, pdk_root=pdk_root, technology_manifest_path=technology_manifest_path, diff --git a/backend/server.py b/backend/server.py index 0b8c3be..7991129 100644 --- a/backend/server.py +++ b/backend/server.py @@ -18,6 +18,13 @@ from werkzeug.security import check_password_hash import database from flask import Response from gds_builder import build_project_gds +from layout_files import ( + LayoutFileError, + is_layout_cell_filename, + load_layout_cell_files, + parse_layout_cell_content, + read_layout_cell_file, +) from pdk_access import ( cleanup_expired_exports, create_export_path, @@ -145,13 +152,17 @@ def cell_routes_path(project_name, cell_name): def write_route_points_sidecar(yaml_content, output_path): """Extract route points from layout YAML and save them beside the cell.""" - layout = yaml.safe_load(yaml_content) or {} + layout = yaml_content if isinstance(yaml_content, dict) else parse_layout_cell_content(yaml_content) routes = {} # The sidecar preserves manually edited route control points separately from # the main YAML file for tooling that wants route-only metadata. for bundle_name, bundle in (layout.get("bundles") or {}).items(): + if not isinstance(bundle, dict): + continue saved_links = [] for link in bundle.get("links") or []: + if not isinstance(link, dict): + continue points = link.get("points") or [] if not points: continue @@ -587,22 +598,24 @@ def list_projects(): os.makedirs(root, exist_ok=True) projects = [] - # Each project is a folder and each YAML file inside that folder is treated - # as one saved cell/canvas. + # Each project is a folder and each valid layout YAML file inside that + # folder is treated as one saved cell/canvas. Route sidecars and malformed + # files are ignored so reopen stays resilient to stale runtime artifacts. for name in sorted(os.listdir(root)): path = os.path.join(root, name) if not os.path.isdir(path): continue cells = [] for filename in sorted(os.listdir(path)): - if not filename.lower().endswith(('.yml', '.yaml')): + if not is_layout_cell_filename(filename): continue cell_name = os.path.splitext(filename)[0] yml_path = os.path.join(path, filename) - cells.append({ - "name": cell_name, - "has_layout": os.path.exists(yml_path) - }) + try: + read_layout_cell_file(yml_path) + except LayoutFileError: + continue + cells.append({"name": cell_name, "has_layout": True}) meta = read_project_meta(name) projects.append({ "name": name, @@ -648,24 +661,20 @@ def get_project(project_name): if not os.path.isdir(root): return jsonify({"error": "Project not found"}), 404 - cells = [] - for filename in sorted(os.listdir(root)): - if not filename.lower().endswith(('.yml', '.yaml')): - continue - cell_name = os.path.splitext(filename)[0] - yml_path = os.path.join(root, filename) - if not os.path.exists(yml_path): - continue - with open(yml_path, 'r', encoding='utf-8') as f: - cells.append({ - "name": cell_name, - "content": f.read() - }) + loaded_cells, warnings = load_layout_cell_files(root) + cells = [ + { + "name": os.path.splitext(cell["filename"])[0], + "content": cell["content"] + } + for cell in loaded_cells + ] return jsonify({ "name": safe_name(project_name, 'project_1'), "cells": cells, - "technology": read_project_meta(project_name).get("technology") + "technology": read_project_meta(project_name).get("technology"), + "warnings": warnings }) @@ -728,18 +737,19 @@ def rename_cell(project_name, cell_name): def save_layout(): """Persist a canvas layout YAML document and refresh its preview assets.""" try: - data = request.get_json() + data = request.get_json(silent=True) or {} project = safe_name(data.get('project'), 'project_1') cell = safe_name(data.get('cell'), 'canvas_1') content = data.get('content', '') create_preview = bool(data.get('preview', True)) + layout_doc = parse_layout_cell_content(content, f"{project}/{cell}.yml") save_path = cell_file_path(project, cell) os.makedirs(os.path.dirname(save_path), exist_ok=True) with open(save_path, 'w', encoding='utf-8') as f: f.write(content) - write_route_points_sidecar(content, cell_routes_path(project, cell)) + write_route_points_sidecar(layout_doc, cell_routes_path(project, cell)) svg_path = None svg_version = None @@ -782,6 +792,8 @@ def save_layout(): "preview_error": preview_error }), 200 + except LayoutFileError as e: + return jsonify({"error": str(e)}), 400 except Exception as e: return jsonify({"error": str(e)}), 500 diff --git a/frontend/canvas.html b/frontend/canvas.html index a3b6cca..39d03d2 100644 --- a/frontend/canvas.html +++ b/frontend/canvas.html @@ -5261,11 +5261,20 @@ Organization : OptiHK Limited } const data = await response.json(); + (data.warnings || []).forEach(warning => addLog(warning)); const technology = data.technology || ''; setProjectTechnology(technology); const manifest = await loadTechnologyManifest(technology); - const knownCompositeNames = new Set((data.cells || []).map(cell => cell.name).filter(name => name !== currentProjectName)); - const parsedCellPages = (data.cells || []).map(cell => pageFromYaml(cell.name, cell.content, manifest, knownCompositeNames)); + const loadedCells = data.cells || []; + const knownCompositeNames = new Set(loadedCells.map(cell => cell.name).filter(name => name !== currentProjectName)); + const parsedCellPages = []; + loadedCells.forEach(cell => { + try { + parsedCellPages.push(pageFromYaml(cell.name, cell.content, manifest, knownCompositeNames)); + } catch (error) { + addLog(`Skipped saved cell "${cell.name}": ${error.message}`); + } + }); const compositeBoxSizes = new Map(parsedCellPages .filter(page => page.type === 'composite') .map(page => [page.name, calculateCompositeBoxSize(page)])); @@ -5327,7 +5336,7 @@ Organization : OptiHK Limited }; loadProject(); - }, [library, currentProjectName, loadTechnologyManifest, toBooleanFlag, makeFreeRouteEdge, buildElementNodesFromYaml, getAvailableComponentsForLoadedComponent, resolveLoadedPinHandle]); + }, [library, currentProjectName, loadTechnologyManifest, toBooleanFlag, makeFreeRouteEdge, buildElementNodesFromYaml, getAvailableComponentsForLoadedComponent, resolveLoadedPinHandle, addLog]); useEffect(() => { if (activePage && activePage.type !== 'layoutPreview' && reactFlowInstance) { diff --git a/tests/layout-backend-static.test.js b/tests/layout-backend-static.test.js index 377be1c..2fd64fa 100644 --- a/tests/layout-backend-static.test.js +++ b/tests/layout-backend-static.test.js @@ -61,7 +61,6 @@ assert( 'save-layout response should include an svg_url for the new layout tab' ); assert( -<<<<<<< HEAD serverPy.includes('svg_ready') && serverPy.includes('svg_version') && serverPy.includes('file_version(svg_path)') && @@ -74,8 +73,6 @@ assert( 'save-layout should publish generated SVG previews atomically instead of serving partially written files' ); assert( -======= ->>>>>>> jingwen_main serverPy.includes('RouterStackUnavailable') && serverPy.includes('except RouterStackUnavailable as e') && serverPy.includes('"preview_status": preview_status') && diff --git a/tests/project-persistence-static.test.js b/tests/project-persistence-static.test.js new file mode 100644 index 0000000..313d341 --- /dev/null +++ b/tests/project-persistence-static.test.js @@ -0,0 +1,54 @@ +/* + * Description: Static regression tests for resilient project save/reopen behavior. + * Inside functions: N/A - assertion-based test/module script. + * Developer : Qin Yue @ 2026 + * Organization : OptiHK Limited + */ +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const root = path.resolve(__dirname, '..'); +const backend = path.join(root, 'backend'); +const layoutFilesPath = path.join(backend, 'layout_files.py'); +const layoutFilesPy = fs.readFileSync(layoutFilesPath, 'utf8'); +const serverPy = fs.readFileSync(path.join(backend, 'server.py'), 'utf8'); +const gdsBuilderPy = fs.readFileSync(path.join(backend, 'gds_builder.py'), 'utf8'); +const routedPreviewPy = fs.readFileSync(path.join(backend, 'routed_layout_preview.py'), 'utf8'); +const canvasHtml = fs.readFileSync(path.join(root, 'frontend', 'canvas.html'), 'utf8'); + +assert( + fs.existsSync(layoutFilesPath), + 'backend/layout_files.py should centralize saved layout YAML filtering and parsing' +); +assert( + layoutFilesPy.includes('ROUTE_SIDECAR_SUFFIXES') && + layoutFilesPy.includes('def is_layout_cell_filename') && + layoutFilesPy.includes('def parse_layout_cell_content') && + layoutFilesPy.includes('def load_layout_cell_files'), + 'layout file helpers should exclude route sidecars and validate saved layout cells' +); +assert( + serverPy.includes('is_layout_cell_filename') && + serverPy.includes('load_layout_cell_files(root)') && + serverPy.includes('parse_layout_cell_content(content') && + serverPy.includes('except LayoutFileError as e'), + 'project list/load/save endpoints should filter invalid YAML and reject malformed saves' +); +assert( + gdsBuilderPy.includes('TemporaryDirectory(prefix="mxpic_gds_project_"') && + gdsBuilderPy.includes('write_layout_cells_to_directory(cells, staged_project_dir)'), + 'Build GDS should call mxpic_router with a clean staged project directory' +); +assert( + routedPreviewPy.includes('staged_project_dir') && + routedPreviewPy.includes('write_layout_cells_to_directory(staged_cells, staged_project_dir)') && + routedPreviewPy.includes('project_dir=staged_project_dir'), + 'Build Layout preview should stage valid cells before calling mxpic_router' +); +assert( + canvasHtml.includes('(data.warnings || []).forEach(warning => addLog(warning))') && + canvasHtml.includes('const parsedCellPages = [];') && + canvasHtml.includes('Skipped saved cell'), + 'canvas project loading should report skipped files and keep loading remaining valid cells' +);