Add encoding CLI arg, retry conversion with UTF-8 on decode failure (#114)

* Add encoding CLI arg, retry conversion with UTF-8 on decode failure, improve typing for convert function, improve filename detection for string arg to convert

* Added bezier test, added output check, added missing specs

* PR comments

* Simplify regex

* write output using encoding flag, harden regex

Author: hbmartin

.github/workflows/lint.yml

@@ -25,17 +25,16 @@ jobs:
       - run: pip install -r requirements.txt
       - run: pip install black ruff pytest pytest-cov
       - run: ruff check .
-      - name: Run mypy if on 3.12, pytype otherwise
+      - name: Run pytype if on 3.11, pyrefly if on 3.14, mypy otherwise
         run: |
           if [[ '${{ steps.cpython3.outputs.python-version }}' == 3.11* ]]; then
             pip install pytype
             pytype -j auto graphviz2drawio
           elif [[ '${{ steps.cpython3.outputs.python-version }}' == 3.14* ]]; then
             echo "Using pyrefly for Python 3.14"
             pip install pyrefly
-            pyrefly check
+            pyrefly check graphviz2drawio
           else
-            echo "Using mypy for Python 3.12"
             pip install mypy
             mypy graphviz2drawio --ignore-missing-imports
           fi

.idea/ryecharm-overrides.xml

@@ -1,7 +1,9 @@
-import pathlib
+#!/usr/bin/env python3
+
 import sys
 from argparse import Namespace
 from io import TextIOWrapper
+from pathlib import Path
 from sys import stderr
 
 from .graphviz2drawio import convert
@@ -12,33 +14,63 @@
 RED_TEXT = "\033[91m"
 BOLD = "\033[1m"
 
+UTF8 = "utf-8"
+
 
 def _gv_filename_to_xml(filename: str) -> str:
     return ".".join(filename.split(".")[:-1]) + ".xml"
 
 
-def _convert_file(to_convert: TextIOWrapper, program: str, outfile: str | None) -> None:
+def _write_stderr_message(to_convert: str) -> None:
+    stderr.write(f"{RED_TEXT}{BOLD}Error converting {to_convert}\n")
+    stderr.write("Please open a report at\n")
+    stderr.write("https://github.com/hbmartin/graphviz2drawio/issues\n")
+    stderr.write("and include your diagram and the following information:\n\n")
+    stderr.write(DEFAULT_TEXT)
+    stderr.write(f"Python: {sys.version}, g2d: {__version__}\n")
+
+
+def _convert_file(
+    to_convert: Path | TextIOWrapper,
+    program: str,
+    encoding: str,
+    outfile: str | None,
+) -> None:
+    output: str | None = None
     try:
-        output = convert(to_convert.read(), program)
-    except BaseException:
-        stderr.write(f"{RED_TEXT}{BOLD}Error converting {to_convert}\n")
-        stderr.write("Please open a report at\n")
-        stderr.write("https://github.com/hbmartin/graphviz2drawio/issues\n")
-        stderr.write("and include your diagram and the following error:\n\n")
-        stderr.write(DEFAULT_TEXT)
-        stderr.write(f"Python: {sys.version}, g2d: {__version__}\n")
+        if isinstance(to_convert, TextIOWrapper):
+            output = convert(to_convert, program)
+        elif isinstance(to_convert, Path):
+            with to_convert.open(encoding=encoding) as contents:
+                output = convert(contents.read(), program)
+    except UnicodeDecodeError:
+        if encoding.lower() != UTF8 and isinstance(to_convert, Path):
+            # Attempt to automatically recover for file. Chinese Windows systems in
+            # particular often use other encodings e.g. gbk, cp950, cp1252, etc. but
+            # the actual dot files are still UTF-8 encoded
+            # https://github.com/hbmartin/graphviz2drawio/issues/105
+            return _convert_file(to_convert, program, UTF8, outfile)
+
+        _write_stderr_message(str(to_convert))
         raise
-    finally:
-        to_convert.close()
+
+    except Exception:
+        _write_stderr_message(str(to_convert))
+        raise
+
+    if output is None:
+        _write_stderr_message(str(to_convert))
+        return None
 
     if outfile is None:
         print(output)
-        return
+        return None
 
-    out_path = pathlib.Path(outfile)
+    out_path = Path(outfile)
     out_path.parent.mkdir(parents=True, exist_ok=True)
-    out_path.write_text(output)
+    out_path.write_text(output, encoding=encoding)
     stderr.write("Converted file: " + outfile + "\n")
+    return None
 
 
 def main() -> None:
@@ -50,7 +82,7 @@ def main() -> None:
     _validate_args(args)
 
     if args.stdout and args.outfile is not None:
-        sys.stdout.write(f"Writing to {args.outfile} (ignoring stdout)\n")
+        sys.stderr.write(f"Writing to {args.outfile} (ignoring stdout)\n")
 
     if len(args.to_convert) == 1:
         in_files = args.to_convert
@@ -60,7 +92,12 @@ def main() -> None:
         out_files = [_gv_filename_to_xml(in_file.name) for in_file in args.to_convert]
 
     for in_file, out_file in zip(in_files, out_files, strict=True):
-        _convert_file(in_file, args.program, out_file)
+        _convert_file(
+            to_convert=in_file,
+            program=args.program,
+            encoding=args.encoding,
+            outfile=out_file,
+        )
 
 
 def _determine_single_output(args: Namespace) -> list[str | None]:

.idea/ryecharm.xml

@@ -1,6 +1,7 @@
-#!/usr/bin/env python3
-
-from typing import IO
+import re
+from io import TextIOBase
+from pathlib import Path
+from typing import TextIO
 
 from pygraphviz import AGraph
 
@@ -9,18 +10,19 @@
 from .mx.MxGraph import MxGraph
 
 
-def convert(graph_to_convert: AGraph | str | IO, layout_prog: str = "dot") -> str:
-    if isinstance(graph_to_convert, AGraph):
-        graph = graph_to_convert
-    else:
-        graph = AGraph(graph_to_convert)
+def convert(
+    graph_to_convert: AGraph | str | TextIOBase | Path | TextIO,
+    layout_prog: str = "dot",
+) -> str:
+    graph = _load_pygraphviz_graph(graph_to_convert)
 
     graph_edges: dict[str, dict] = {
         f"{e[0]}->{e[1]}-"
         # pyrefly: ignore  # missing-attribute
         + (e.attr.get("xlabel") or e.attr.get("label") or ""): e.attr.to_dict()
         for e in graph.edges_iter()
     }
+
     # pyrefly: ignore  # missing-attribute
     graph_nodes: dict[str, dict] = {n: n.attr.to_dict() for n in graph.nodes_iter()}
 
@@ -45,3 +47,26 @@ def convert(graph_to_convert: AGraph | str | IO, layout_prog: str = "dot") -> st
     # Put clusters first, so that nodes are drawn in front
     mx_graph = MxGraph(clusters, nodes, edges)
     return mx_graph.value()
+
+
+def _load_pygraphviz_graph(
+    graph_to_convert: AGraph | str | TextIOBase | Path | TextIO,
+) -> AGraph:
+    if isinstance(graph_to_convert, AGraph):
+        return graph_to_convert
+    if isinstance(graph_to_convert, str):
+        # This fixes a pygraphviz bug where a string beginning with a comment
+        # is mistakenly identified as a filename.
+        # https://github.com/pygraphviz/pygraphviz/issues/536
+        pattern = re.compile(
+            pattern=r"^(?=(\s*))\1(strict)?(?=(\s*))\3(graph|digraph)[^{]*{",
+            flags=re.MULTILINE,
+        )
+        if pattern.search(graph_to_convert):
+            return AGraph(string=graph_to_convert)
+        return AGraph(filename=graph_to_convert)
+    # pyrefly: ignore  # missing-attribute
+    if hasattr(graph_to_convert, "read") and callable(graph_to_convert.read):
+        return AGraph(string=graph_to_convert.read())
+    # Use builtin type detection which includes:  hasattr(thing, "open")
+    return AGraph(graph_to_convert)

graphviz2drawio/__main__.py

@@ -1,5 +1,15 @@
-from argparse import ArgumentParser, FileType
+import sys
+from argparse import ArgumentParser
+from locale import getpreferredencoding
+from pathlib import Path
 from sys import stdin
+from typing import TextIO
+
+
+class NonOpeningFileType:
+    def __call__(self, string: str) -> TextIO | Path:
+        # the special argument "-" means sys.std{in,out}
+        return sys.stdin if string == "-" else Path(string)
 
 
 class Arguments(ArgumentParser):
@@ -13,7 +23,7 @@ def __init__(self, version: str) -> None:
             metavar="file(s).dot",
             help="Path of the graphviz file(s) to convert (or stdin).",
             nargs="*",
-            type=FileType("r"),
+            type=NonOpeningFileType(),
             default=[stdin],
         )
         self.add_argument(
@@ -34,9 +44,16 @@ def __init__(self, version: str) -> None:
         self.add_argument(
             "-p",
             "--program",
-            help="layout program (defaults to dot)",
+            help="Layout program (defaults to dot)",
             default="dot",
         )
+        self.add_argument(
+            "--encoding",
+            "-e",
+            type=str,
+            default=getpreferredencoding(do_setlocale=False).lower(),
+            help="Encoding to use when opening files (default: %(default)s)",
+        )
         self.add_argument(
             "--version",
             action="version",

graphviz2drawio/graphviz2drawio.py

@@ -10,7 +10,7 @@ line-length = 88
 indent-width = 4
 
 lint.select = ["ALL"]
-lint.ignore = ["D100", "D101", "D102", "D103", "D104", "D105", "D107", "D203", "D213", "ERA001", "ICN001", "PLR0913", "S314", "SIM102", "TD002", "TID252", "N999"]
+lint.ignore = ["D100", "D101", "D102", "D103", "D104", "D105", "D107", "D203", "D213", "ERA001", "ICN001", "PLR0913", "PT009", "S314", "SIM102", "TD002", "TID252", "N999"]
 target-version = "py310"
 
 [tool.ruff.format]