Copilot Chat Conversation Export: Inline Form XObjects in PDF using

#!/usr/bin/env python3
"""
Inline simple Form XObjects in PDF page content streams.

Requirements:
  pip install pikepdf

Usage:
  python inline_form_xobjects.py input.pdf output.pdf

Notes:
- Handles direct `/Name Do` calls where `/Name` resolves to a Form XObject in page /Resources /XObject.
- Recurses into nested Form XObjects.
- Wraps inlined form content in `q ... Q` and applies the form /Matrix with `cm` if present.
- Merges form resources into page resources with auto-renaming to avoid key collisions.
- This is intentionally small and pragmatic, not a full PDF interpreter.
"""

import sys
import re
from typing import Dict, Tuple, Optional, Set

import pikepdf
from pikepdf import Name, Dictionary, Stream, Array


# ---------- basic tokenization helpers (lightweight, not full PDF parser) ----------

_WHITESPACE = b" \t\r\n\x0c\x00"


def _skip_ws(data: bytes, i: int) -> int:
    n = len(data)
    while i < n and data[i] in _WHITESPACE:
        i += 1
    return i


def _read_name(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] must be '/'
    j = i + 1
    n = len(data)
    while j < n and data[j] not in _WHITESPACE + b"[]<>()/%":
        j += 1
    return data[i:j], j


def _read_word(data: bytes, i: int) -> Tuple[bytes, int]:
    j = i
    n = len(data)
    while j < n and data[j] not in _WHITESPACE:
        # stop at delimiters commonly separating tokens
        if data[j] in b"[]<>()/":
            break
        j += 1
    if j == i and i < n and data[i] in b"[]<>()":
        return data[i:i+1], i + 1
    return data[i:j], j


# ---------- resource merge helpers ----------

RESOURCE_CATEGORIES = [
    Name("/ExtGState"),
    Name("/ColorSpace"),
    Name("/Pattern"),
    Name("/Shading"),
    Name("/XObject"),
    Name("/Font"),
    Name("/Properties"),
]


def ensure_dict(obj, pdf) -> Dictionary:
    if obj is None:
        return Dictionary()
    if isinstance(obj, Dictionary):
        return obj
    # fallback
    return Dictionary(obj)


def _unique_name(target_dict: Dictionary, base: Name) -> Name:
    if base not in target_dict:
        return base
    base_str = str(base)  # "/F1"
    k = 1
    while True:
        cand = Name(base_str + f"_inl{k}")
        if cand not in target_dict:
            return cand
        k += 1


def merge_resources_with_rename(
    page_resources: Dictionary,
    form_resources: Dictionary,
) -> Dict[Name, Name]:
    """
    Merge form_resources into page_resources.
    Returns name remap dict for *all* categories:
      old Name -> new Name
    """
    remap: Dict[Name, Name] = {}

    for cat in RESOURCE_CATEGORIES:
        src_cat = form_resources.get(cat, None)
        if not isinstance(src_cat, Dictionary):
            continue

        dst_cat = page_resources.get(cat, None)
        if not isinstance(dst_cat, Dictionary):
            dst_cat = Dictionary()
            page_resources[cat] = dst_cat

        for old_name, val in src_cat.items():
            old_name = Name(old_name)
            if old_name in dst_cat:
                # identical object? keep existing mapping
                try:
                    if dst_cat[old_name].objgen == val.objgen:
                        remap[old_name] = old_name
                        continue
                except Exception:
                    pass
                new_name = _unique_name(dst_cat, old_name)
            else:
                new_name = old_name

            dst_cat[new_name] = val
            remap[old_name] = new_name

    return remap


def remap_names_in_content(content: bytes, remap: Dict[Name, Name]) -> bytes:
    """
    Lightweight lexical rename of name tokens, good enough for content streams.
    """
    if not remap:
        return content

    out = bytearray()
    i = 0
    n = len(content)

    while i < n:
        b = content[i]
        if b == ord('/'):
            tok, j = _read_name(content, i)  # b"/F1"
            old = Name(tok.decode("latin1"))
            new = remap.get(old, old)
            out.extend(str(new).encode("latin1"))
            i = j
        else:
            out.append(b)
            i += 1

    return bytes(out)


# ---------- form inlining ----------

def get_stream_bytes(obj: Stream) -> bytes:
    return obj.read_bytes()


def get_form_matrix(form_obj: Stream) -> Optional[Array]:
    m = form_obj.get(Name("/Matrix"), None)
    if isinstance(m, Array) and len(m) == 6:
        return m
    return None


def is_form_xobject(obj) -> bool:
    if not isinstance(obj, Stream):
        return False
    st = obj.get(Name("/Subtype"), None)
    ty = obj.get(Name("/Type"), None)
    return st == Name("/Form") or (ty == Name("/XObject") and st == Name("/Form"))


def inline_content_stream(
    pdf: pikepdf.Pdf,
    content: bytes,
    page_resources: Dictionary,
    local_resources: Dictionary,
    visited_objgens: Set[Tuple[int, int]],
) -> bytes:
    """
    Inline `/Name Do` when Name references Form XObject in local_resources/XObject.
    """
    xobjs = local_resources.get(Name("/XObject"), None)
    if not isinstance(xobjs, Dictionary):
        return content

    out = bytearray()
    i = 0
    n = len(content)

    while i < n:
        i0 = i
        i = _skip_ws(content, i)
        out.extend(content[i0:i])  # preserve ws

        if i >= n:
            break

        # try read name token
        if content[i] == ord('/'):
            name_tok, j = _read_name(content, i)  # /Fm1
            k = _skip_ws(content, j)
            op, k2 = _read_word(content, k)      # maybe Do

            if op == b"Do":
                xname = Name(name_tok.decode("latin1"))
                xobj = xobjs.get(xname, None)
                if xobj is not None and is_form_xobject(xobj):
                    # recursion guard
                    objgen = getattr(xobj, "objgen", None)
                    if objgen and objgen in visited_objgens:
                        # cyclic ref; leave as-is
                        out.extend(content[i:k2])
                        i = k2
                        continue
                    if objgen:
                        visited_objgens.add(objgen)

                    # form resources
                    form_resources = xobj.get(Name("/Resources"), None)
                    if not isinstance(form_resources, Dictionary):
                        form_resources = Dictionary()

                    # merge resources into page, build remap
                    remap = merge_resources_with_rename(page_resources, form_resources)

                    # recurse into form content
                    form_bytes = get_stream_bytes(xobj)
                    inlined_inner = inline_content_stream(
                        pdf=pdf,
                        content=form_bytes,
                        page_resources=page_resources,
                        local_resources=form_resources,
                        visited_objgens=visited_objgens,
                    )

                    # remap resource names lexically
                    inlined_inner = remap_names_in_content(inlined_inner, remap)

                    # apply form matrix
                    matrix = get_form_matrix(xobj)
                    block = bytearray()
                    block.extend(b"q\n")
                    if matrix is not None:
                        nums = " ".join(str(float(v)).rstrip("0").rstrip(".") if isinstance(v, float) else str(v) for v in matrix)
                        block.extend(nums.encode("latin1"))
                        block.extend(b" cm\n")
                    block.extend(inlined_inner)
                    if not inlined_inner.endswith(b"\n"):
                        block.extend(b"\n")
                    block.extend(b"Q\n")

                    out.extend(block)
                    i = k2
                    if objgen:
                        visited_objgens.discard(objgen)
                    continue

            # not a Form Do -> copy token and continue from j
            out.extend(content[i:j])
            i = j
            continue

        # normal token/char
        tok, j = _read_word(content, i)
        if j == i:  # safety
            out.append(content[i])
            i += 1
        else:
            out.extend(content[i:j])
            i = j

    return bytes(out)


def page_content_bytes(page) -> bytes:
    """
    Return concatenated content stream bytes for a page.
    """
    c = page.get(Name("/Contents"), None)
    if c is None:
        return b""
    if isinstance(c, Stream):
        return c.read_bytes()
    if isinstance(c, Array):
        parts = []
        for s in c:
            if isinstance(s, Stream):
                parts.append(s.read_bytes())
        return b"\n".join(parts)
    return b""


def set_page_content_bytes(pdf: pikepdf.Pdf, page, data: bytes) -> None:
    page[Name("/Contents")] = pdf.make_stream(data)


def process_pdf(infile: str, outfile: str) -> None:
    with pikepdf.open(infile) as pdf:
        for page in pdf.pages:
            page_resources = page.get(Name("/Resources"), None)
            if not isinstance(page_resources, Dictionary):
                page_resources = Dictionary()
                page[Name("/Resources")] = page_resources

            original = page_content_bytes(page)
            if not original:
                continue

            rewritten = inline_content_stream(
                pdf=pdf,
                content=original,
                page_resources=page_resources,
                local_resources=page_resources,
                visited_objgens=set(),
            )

            if rewritten != original:
                set_page_content_bytes(pdf, page, rewritten)

        pdf.save(outfile)


def main():
    if len(sys.argv) != 3:
        print("Usage: python inline_form_xobjects.py input.pdf output.pdf")
        sys.exit(2)
    infile, outfile = sys.argv[1], sys.argv[2]
    process_pdf(infile, outfile)
    print(f"Wrote {outfile}")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Remove manual kerning tweaks from PDF text-show operators by rewriting TJ arrays.

What it does:
- Finds TJ arrays like: [(A) -120 (V) 30 (A)] TJ
- Removes all numeric adjustments, keeping only text strings:
  [(A) (V) (A)] TJ
- Optionally converts single-element TJ arrays to Tj:
  [(Hello)] TJ  -> (Hello) Tj

What it does NOT do:
- Full PDF syntax parsing (this is a pragmatic tokenizer)
- Preserve exact original spacing from kerning numbers (visual text may reflow slightly)

Usage:
  python remove_manual_kerning.py in.pdf out.pdf
  python remove_manual_kerning.py in.pdf out.pdf --keep-tj
"""

import sys
from typing import Tuple, List
import pikepdf
from pikepdf import Name, Array, Stream

WS = b" \t\r\n\x0c\x00"


def skip_ws(data: bytes, i: int) -> int:
    n = len(data)
    while i < n and data[i] in WS:
        i += 1
    return i


def read_number_token(data: bytes, i: int) -> Tuple[bytes, int]:
    j = i
    n = len(data)
    if j < n and data[j] in b"+-":
        j += 1
    dot_seen = False
    while j < n:
        c = data[j]
        if c >= ord('0') and c <= ord('9'):
            j += 1
            continue
        if c == ord('.') and not dot_seen:
            dot_seen = True
            j += 1
            continue
        break
    return data[i:j], j


def is_number_token(tok: bytes) -> bool:
    if not tok:
        return False
    try:
        float(tok.decode("latin1"))
        return True
    except Exception:
        return False


def read_literal_string(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '('
    n = len(data)
    j = i + 1
    depth = 1
    out = bytearray()
    out.append(ord('('))
    while j < n and depth > 0:
        c = data[j]
        out.append(c)
        if c == ord('\\'):
            # escape next byte if present
            j += 1
            if j < n:
                out.append(data[j])
        elif c == ord('('):
            depth += 1
        elif c == ord(')'):
            depth -= 1
        j += 1
    return bytes(out), j


def read_hex_string(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '<' and next != '<'
    n = len(data)
    j = i + 1
    while j < n and data[j] != ord('>'):
        j += 1
    if j < n:
        j += 1
    return data[i:j], j


def read_array(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '[' ; returns full bracketed bytes
    n = len(data)
    j = i + 1
    depth = 1
    out = bytearray()
    out.append(ord('['))

    while j < n and depth > 0:
        c = data[j]

        if c == ord('('):
            s, j2 = read_literal_string(data, j)
            out.extend(s)
            j = j2
            continue
        if c == ord('<'):
            # could be hex string or dictionary start <<
            if j + 1 < n and data[j + 1] == ord('<'):
                # dictionary marker inside array, copy literally
                out.append(c)
                j += 1
                out.append(data[j])
                j += 1
                continue
            hs, j2 = read_hex_string(data, j)
            out.extend(hs)
            j = j2
            continue

        out.append(c)
        if c == ord('['):
            depth += 1
        elif c == ord(']'):
            depth -= 1
        j += 1

    return bytes(out), j


def tokenize_array_items(arr: bytes) -> List[bytes]:
    # arr includes '[' ... ']'
    i = 1
    n = len(arr) - 1
    items = []
    while i < n:
        i = skip_ws(arr, i)
        if i >= n:
            break
        c = arr[i]
        if c == ord('('):
            s, j = read_literal_string(arr, i)
            items.append(s)
            i = j
        elif c == ord('<'):
            if i + 1 < n and arr[i + 1] == ord('<'):
                # dictionary token, keep raw until matching >> heuristically
                j = i + 2
                while j + 1 < n and not (arr[j] == ord('>') and arr[j + 1] == ord('>')):
                    j += 1
                j = min(j + 2, n)
                items.append(arr[i:j])
                i = j
            else:
                hs, j = read_hex_string(arr, i)
                items.append(hs)
                i = j
        elif c == ord('/'):
            j = i + 1
            while j < n and arr[j] not in WS + b"[]<>()/%":
                j += 1
            items.append(arr[i:j])
            i = j
        else:
            tok, j = read_number_token(arr, i)
            if tok:
                items.append(tok)
                i = j
            else:
                # fallback generic token
                j = i
                while j < n and arr[j] not in WS + b"[]":
                    j += 1
                items.append(arr[i:j])
                i = j
    return items


def rewrite_tj_array(arr: bytes, keep_tj: bool) -> bytes:
    items = tokenize_array_items(arr)
    kept = []
    for t in items:
        t_strip = t.strip()
        if not t_strip:
            continue
        # Keep only strings (literal or hex)
        if t_strip.startswith(b"(") or (t_strip.startswith(b"<") and not t_strip.startswith(b"<<")):
            kept.append(t_strip)
        elif is_number_token(t_strip):
            # drop kerning adjustment
            continue
        else:
            # unknown token in TJ array: keep conservatively
            kept.append(t_strip)

    if not kept:
        # empty text; keep a harmless empty string
        kept = [b"()"]

    if (not keep_tj) and len(kept) == 1:
        # convert to Tj operand form; caller appends operator
        return kept[0] + b" Tj"

    return b"[ " + b" ".join(kept) + b" ] TJ"


def rewrite_content_stream(data: bytes, keep_tj: bool) -> bytes:
    out = bytearray()
    i = 0
    n = len(data)

    while i < n:
        c = data[i]
        if c == ord('['):
            arr, j = read_array(data, i)
            k = skip_ws(data, j)

            # read operator token after array
            op_start = k
            while k < n and data[k] not in WS:
                if data[k] in b"[]<>()":
                    break
                k += 1
            op = data[op_start:k]

            if op == b"TJ":
                rewritten = rewrite_tj_array(arr, keep_tj=keep_tj)
                out.extend(rewritten)
                i = k
                continue
            else:
                out.extend(arr)
                i = j
                continue

        out.append(c)
        i += 1

    return bytes(out)


def page_content_bytes(page) -> bytes:
    c = page.get(Name("/Contents"), None)
    if c is None:
        return b""
    if isinstance(c, Stream):
        return c.read_bytes()
    if isinstance(c, Array):
        parts = []
        for s in c:
            if isinstance(s, Stream):
                parts.append(s.read_bytes())
        return b"\n".join(parts)
    return b""


def set_page_content_bytes(pdf: pikepdf.Pdf, page, data: bytes) -> None:
    page[Name("/Contents")] = pdf.make_stream(data)


def process_pdf(infile: str, outfile: str, keep_tj: bool) -> None:
    with pikepdf.open(infile) as pdf:
        changed = 0
        for page in pdf.pages:
            original = page_content_bytes(page)
            if not original:
                continue
            rewritten = rewrite_content_stream(original, keep_tj=keep_tj)
            if rewritten != original:
                set_page_content_bytes(pdf, page, rewritten)
                changed += 1
        pdf.save(outfile)
    print(f"Wrote {outfile} (modified pages: {changed})")


def main():
    if len(sys.argv) < 3:
        print("Usage: python remove_manual_kerning.py in.pdf out.pdf [--keep-tj]")
        sys.exit(2)

    infile = sys.argv[1]
    outfile = sys.argv[2]
    keep_tj = "--keep-tj" in sys.argv[3:]

    process_pdf(infile, outfile, keep_tj=keep_tj)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Remove manual kerning tweaks from PDF text-show operators by rewriting TJ arrays,
including inside referenced Form XObjects (recursively).

Requirements:
  pip install pikepdf

Usage:
  python remove_manual_kerning_all_forms.py in.pdf out.pdf
  python remove_manual_kerning_all_forms.py in.pdf out.pdf --keep-tj
"""

import sys
from typing import Tuple, List, Set

import pikepdf
from pikepdf import Name, Array, Stream, Dictionary

WS = b" \t\r\n\x0c\x00"


# ---------------- tokenizer helpers ----------------

def skip_ws(data: bytes, i: int) -> int:
    n = len(data)
    while i < n and data[i] in WS:
        i += 1
    return i


def read_number_token(data: bytes, i: int) -> Tuple[bytes, int]:
    j = i
    n = len(data)
    if j < n and data[j] in b"+-":
        j += 1
    dot_seen = False
    digit_seen = False
    while j < n:
        c = data[j]
        if ord('0') <= c <= ord('9'):
            digit_seen = True
            j += 1
            continue
        if c == ord('.') and not dot_seen:
            dot_seen = True
            j += 1
            continue
        break
    if not digit_seen:
        return b"", i
    return data[i:j], j


def is_number_token(tok: bytes) -> bool:
    if not tok:
        return False
    try:
        float(tok.decode("latin1"))
        return True
    except Exception:
        return False


def read_literal_string(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '('
    n = len(data)
    j = i + 1
    depth = 1
    out = bytearray(b"(")
    while j < n and depth > 0:
        c = data[j]
        out.append(c)
        if c == ord('\\'):
            j += 1
            if j < n:
                out.append(data[j])
        elif c == ord('('):
            depth += 1
        elif c == ord(')'):
            depth -= 1
        j += 1
    return bytes(out), j


def read_hex_string(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '<' and next != '<'
    n = len(data)
    j = i + 1
    while j < n and data[j] != ord('>'):
        j += 1
    if j < n:
        j += 1
    return data[i:j], j


def read_array(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '['
    n = len(data)
    j = i + 1
    depth = 1
    out = bytearray(b"[")
    while j < n and depth > 0:
        c = data[j]
        if c == ord('('):
            s, j2 = read_literal_string(data, j)
            out.extend(s)
            j = j2
            continue
        if c == ord('<'):
            if j + 1 < n and data[j + 1] == ord('<'):
                out.extend(b"<<")
                j += 2
                continue
            hs, j2 = read_hex_string(data, j)
            out.extend(hs)
            j = j2
            continue

        out.append(c)
        if c == ord('['):
            depth += 1
        elif c == ord(']'):
            depth -= 1
        j += 1
    return bytes(out), j


def tokenize_array_items(arr: bytes) -> List[bytes]:
    # arr includes [ ... ]
    i = 1
    n = len(arr) - 1
    items: List[bytes] = []

    while i < n:
        i = skip_ws(arr, i)
        if i >= n:
            break
        c = arr[i]

        if c == ord('('):
            s, j = read_literal_string(arr, i)
            items.append(s)
            i = j
        elif c == ord('<'):
            if i + 1 < n and arr[i + 1] == ord('<'):
                # dictionary token inside array (rare for TJ arrays)
                j = i + 2
                while j + 1 < n and not (arr[j] == ord('>') and arr[j + 1] == ord('>')):
                    j += 1
                j = min(j + 2, n)
                items.append(arr[i:j])
                i = j
            else:
                hs, j = read_hex_string(arr, i)
                items.append(hs)
                i = j
        elif c == ord('/'):
            j = i + 1
            while j < n and arr[j] not in WS + b"[]<>()/%":
                j += 1
            items.append(arr[i:j])
            i = j
        else:
            tok, j = read_number_token(arr, i)
            if tok:
                items.append(tok)
                i = j
            else:
                j = i
                while j < n and arr[j] not in WS + b"[]":
                    j += 1
                items.append(arr[i:j])
                i = j

    return items


# ---------------- TJ rewrite ----------------

def rewrite_tj_array(arr: bytes, keep_tj: bool) -> bytes:
    items = tokenize_array_items(arr)
    kept: List[bytes] = []

    for t in items:
        s = t.strip()
        if not s:
            continue
        # Keep text strings only; drop numeric kerning adjustments
        if s.startswith(b"(") or (s.startswith(b"<") and not s.startswith(b"<<")):
            kept.append(s)
        elif is_number_token(s):
            continue
        else:
            # conservative: keep unknown tokens
            kept.append(s)

    if not kept:
        kept = [b"()"]

    if (not keep_tj) and len(kept) == 1:
        return kept[0] + b" Tj"
    return b"[ " + b" ".join(kept) + b" ] TJ"


def rewrite_content_stream(data: bytes, keep_tj: bool) -> bytes:
    out = bytearray()
    i = 0
    n = len(data)

    while i < n:
        if data[i] == ord('['):
            arr, j = read_array(data, i)
            k = skip_ws(data, j)

            op_start = k
            while k < n and data[k] not in WS and data[k] not in b"[]<>()":
                k += 1
            op = data[op_start:k]

            if op == b"TJ":
                out.extend(rewrite_tj_array(arr, keep_tj=keep_tj))
                i = k
                continue
            else:
                out.extend(arr)
                i = j
                continue

        out.append(data[i])
        i += 1

    return bytes(out)


# ---------------- PDF traversal ----------------

def stream_bytes(obj: Stream) -> bytes:
    return obj.read_bytes()


def set_stream_bytes(pdf: pikepdf.Pdf, obj: Stream, data: bytes) -> None:
    # Replace stream data while keeping dictionary entries.
    obj.write(data)


def is_form_xobject(obj) -> bool:
    if not isinstance(obj, Stream):
        return False
    subtype = obj.get(Name("/Subtype"), None)
    typ = obj.get(Name("/Type"), None)
    return subtype == Name("/Form") or (typ == Name("/XObject") and subtype == Name("/Form"))


def process_form_xobject_recursive(
    pdf: pikepdf.Pdf,
    form_obj: Stream,
    keep_tj: bool,
    visited: Set[tuple],
) -> int:
    """
    Process this form's stream, then recurse into its /Resources /XObject forms.
    Returns number of modified streams.
    """
    changed = 0
    objgen = getattr(form_obj, "objgen", None)
    if objgen is not None:
        if objgen in visited:
            return 0
        visited.add(objgen)

    # Rewrite this form stream
    original = stream_bytes(form_obj)
    rewritten = rewrite_content_stream(original, keep_tj=keep_tj)
    if rewritten != original:
        set_stream_bytes(pdf, form_obj, rewritten)
        changed += 1

    # Recurse into nested forms referenced by this form's resources
    resources = form_obj.get(Name("/Resources"), None)
    if isinstance(resources, Dictionary):
        changed += process_forms_in_resources(pdf, resources, keep_tj, visited)

    return changed


def process_forms_in_resources(
    pdf: pikepdf.Pdf,
    resources: Dictionary,
    keep_tj: bool,
    visited: Set[tuple],
) -> int:
    changed = 0
    xobj_dict = resources.get(Name("/XObject"), None)
    if not isinstance(xobj_dict, Dictionary):
        return 0

    for _name, xobj in xobj_dict.items():
        if is_form_xobject(xobj):
            changed += process_form_xobject_recursive(pdf, xobj, keep_tj, visited)

    return changed


def page_content_bytes(page) -> bytes:
    c = page.get(Name("/Contents"), None)
    if c is None:
        return b""
    if isinstance(c, Stream):
        return c.read_bytes()
    if isinstance(c, Array):
        parts = []
        for s in c:
            if isinstance(s, Stream):
                parts.append(s.read_bytes())
        return b"\n".join(parts)
    return b""


def set_page_content_bytes(pdf: pikepdf.Pdf, page, data: bytes) -> None:
    page[Name("/Contents")] = pdf.make_stream(data)


def process_pdf(infile: str, outfile: str, keep_tj: bool) -> None:
    with pikepdf.open(infile) as pdf:
        modified_streams = 0
        visited_forms: Set[tuple] = set()

        for page in pdf.pages:
            # 1) rewrite page content
            original = page_content_bytes(page)
            if original:
                rewritten = rewrite_content_stream(original, keep_tj=keep_tj)
                if rewritten != original:
                    set_page_content_bytes(pdf, page, rewritten)
                    modified_streams += 1

            # 2) rewrite all reachable form xobjects from page resources
            resources = page.get(Name("/Resources"), None)
            if isinstance(resources, Dictionary):
                modified_streams += process_forms_in_resources(
                    pdf, resources, keep_tj, visited_forms
                )

        pdf.save(outfile)
    print(f"Wrote {outfile} (modified streams: {modified_streams})")


def main():
    if len(sys.argv) < 3:
        print("Usage: python remove_manual_kerning_all_forms.py in.pdf out.pdf [--keep-tj]")
        sys.exit(2)

    infile = sys.argv[1]
    outfile = sys.argv[2]
    keep_tj = "--keep-tj" in sys.argv[3:]

    process_pdf(infile, outfile, keep_tj=keep_tj)


if __name__ == "__main__":
    main()

def _is_literal_string_token(tok: bytes) -> bool:
    return tok.startswith(b"(") and tok.endswith(b")")


def _is_hex_string_token(tok: bytes) -> bool:
    return tok.startswith(b"<") and tok.endswith(b">") and not tok.startswith(b"<<")


def _decode_pdf_literal(tok: bytes) -> bytes:
    """
    Decode a PDF literal string token ( ... ) into raw bytes.
    Handles balanced parens/escapes as they appear in tokenized literals.
    """
    assert _is_literal_string_token(tok)
    s = tok[1:-1]
    out = bytearray()
    i = 0
    n = len(s)

    while i < n:
        c = s[i]
        if c != 0x5C:  # backslash
            out.append(c)
            i += 1
            continue

        i += 1
        if i >= n:
            out.append(0x5C)
            break

        e = s[i]
        i += 1

        # Simple escapes
        mapping = {
            ord('n'): 0x0A,
            ord('r'): 0x0D,
            ord('t'): 0x09,
            ord('b'): 0x08,
            ord('f'): 0x0C,
            ord('('): ord('('),
            ord(')'): ord(')'),
            ord('\\'): ord('\\'),
        }
        if e in mapping:
            out.append(mapping[e])
            continue

        # Line continuation: backslash + EOL => ignore both
        if e == 0x0D:  # \r
            if i < n and s[i] == 0x0A:
                i += 1
            continue
        if e == 0x0A:  # \n
            continue

        # Octal escape: up to 3 octal digits (already consumed first digit e)
        if 0x30 <= e <= 0x37:
            oct_digits = [e]
            for _ in range(2):
                if i < n and 0x30 <= s[i] <= 0x37:
                    oct_digits.append(s[i])
                    i += 1
                else:
                    break
            out.append(int(bytes(oct_digits).decode("latin1"), 8) & 0xFF)
            continue

        # Unknown escape => literal char
        out.append(e)

    return bytes(out)


def _encode_pdf_literal(raw: bytes) -> bytes:
    """
    Encode raw bytes into a safe PDF literal string token ( ... ).
    Escapes backslash and parentheses; uses common escapes for controls.
    """
    out = bytearray(b"(")
    for b in raw:
        if b == 0x5C:      # \
            out.extend(b"\\\\")
        elif b == 0x28:    # (
            out.extend(b"\\(")
        elif b == 0x29:    # )
            out.extend(b"\\)")
        elif b == 0x0A:
            out.extend(b"\\n")
        elif b == 0x0D:
            out.extend(b"\\r")
        elif b == 0x09:
            out.extend(b"\\t")
        elif b == 0x08:
            out.extend(b"\\b")
        elif b == 0x0C:
            out.extend(b"\\f")
        elif b < 32 or b > 126:
            out.extend(f"\\{b:03o}".encode("latin1"))
        else:
            out.append(b)
    out.extend(b")")
    return bytes(out)


def _normalize_hex_payload(payload: bytes) -> bytes:
    # Remove whitespace inside hex string and uppercase
    payload = b"".join(payload.split()).upper()
    if len(payload) % 2 == 1:
        payload += b"0"  # PDF rule: odd nibble implicitly padded
    return payload


def _merge_adjacent_string_tokens(tokens: list[bytes]) -> list[bytes]:
    """
    Merge adjacent string tokens:
      - literal + literal => one literal
      - hex + hex => one hex
    Keep literal/hex boundaries separate (no cross-type conversion).
    """
    merged: list[bytes] = []
    i = 0
    n = len(tokens)

    while i < n:
        t = tokens[i]

        if _is_literal_string_token(t):
            raw = bytearray(_decode_pdf_literal(t))
            j = i + 1
            while j < n and _is_literal_string_token(tokens[j]):
                raw.extend(_decode_pdf_literal(tokens[j]))
                j += 1
            merged.append(_encode_pdf_literal(bytes(raw)))
            i = j
            continue

        if _is_hex_string_token(t):
            payload = _normalize_hex_payload(t[1:-1])
            j = i + 1
            while j < n and _is_hex_string_token(tokens[j]):
                payload += _normalize_hex_payload(tokens[j][1:-1])
                j += 1
            merged.append(b"<" + payload + b">")
            i = j
            continue

        merged.append(t)
        i += 1

    return merged


def rewrite_tj_array(arr: bytes, keep_tj: bool) -> bytes:
    items = tokenize_array_items(arr)

    # 1) Remove numeric kerning adjustments, keep string operands
    kept: list[bytes] = []
    for t in items:
        s = t.strip()
        if not s:
            continue

        if _is_literal_string_token(s) or _is_hex_string_token(s):
            kept.append(s)
        elif is_number_token(s):
            # remove kerning tweak
            continue
        else:
            # conservative fallback
            kept.append(s)

    if not kept:
        kept = [b"()"]

    # 2) Merge adjacent strings into one literal/hex token
    kept = _merge_adjacent_string_tokens(kept)

    # 3) Optionally collapse single-element TJ -> Tj
    if (not keep_tj) and len(kept) == 1:
        return kept[0] + b" Tj"

    return b"[ " + b" ".join(kept) + b" ] TJ"

#!/usr/bin/env python3
"""
Remove marked-content wrappers (BMC/BDC ... EMC) from PDF content streams.

It processes:
- Page /Contents streams
- Form XObject streams in /Resources /XObject (recursively)

Wrappers removed:
- /Tag BMC ... EMC
- /Tag <<...>> BDC ... EMC
- /Tag /PropName BDC ... EMC
- /Tag <</MCID ...>> BDC ... EMC
(and nested combinations)

Preserves the enclosed operators/content bytes as much as possible.

Requirements:
  pip install pikepdf

Usage:
  python remove_marked_content_wrappers.py in.pdf out.pdf
"""

import sys
from typing import Set, Tuple
import pikepdf
from pikepdf import Name, Stream, Array, Dictionary

WS = b" \t\r\n\x0c\x00"


# ---------------- token readers ----------------

def skip_ws(data: bytes, i: int) -> int:
    n = len(data)
    while i < n and data[i] in WS:
        i += 1
    return i


def read_name(data: bytes, i: int) -> Tuple[bytes, int]:
    # expects data[i] == '/'
    j = i + 1
    n = len(data)
    while j < n and data[j] not in WS + b"[]<>()/%":
        j += 1
    return data[i:j], j


def read_word(data: bytes, i: int) -> Tuple[bytes, int]:
    j = i
    n = len(data)
    while j < n and data[j] not in WS + b"[]<>()/%":
        j += 1
    return data[i:j], j


def read_literal_string(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '('
    out = bytearray()
    n = len(data)
    depth = 0
    j = i
    while j < n:
        c = data[j]
        out.append(c)
        if c == ord('('):
            depth += 1
        elif c == ord(')'):
            depth -= 1
            if depth == 0:
                j += 1
                break
        elif c == ord('\\'):
            j += 1
            if j < n:
                out.append(data[j])
        j += 1
    return bytes(out), j


def read_hex_string(data: bytes, i: int) -> Tuple[bytes, int]:
    # data[i] == '<' and next != '<'
    j = i + 1
    n = len(data)
    while j < n and data[j] != ord('>'):
        j += 1
    if j < n:
        j += 1
    return data[i:j], j


def read_dict(data: bytes, i: int) -> Tuple[bytes, int]:
    # reads << ... >> including nested dictionaries and strings
    assert i + 1 < len(data) and data[i] == ord('<') and data[i + 1] == ord('<')
    out = bytearray(b"<<")
    j = i + 2
    n = len(data)
    depth = 1

    while j < n and depth > 0:
        c = data[j]

        if c == ord('('):
            s, j2 = read_literal_string(data, j)
            out.extend(s)
            j = j2
            continue
        if c == ord('<'):
            if j + 1 < n and data[j + 1] == ord('<'):
                out.extend(b"<<")
                j += 2
                depth += 1
                continue
            hs, j2 = read_hex_string(data, j)
            out.extend(hs)
            j = j2
            continue
        if c == ord('>') and j + 1 < n and data[j + 1] == ord('>'):
            out.extend(b">>")
            j += 2
            depth -= 1
            continue

        out.append(c)
        j += 1

    return bytes(out), j


def skip_single_operand(data: bytes, i: int) -> int:
    i = skip_ws(data, i)
    if i >= len(data):
        return i
    c = data[i]
    if c == ord('/'):
        _, j = read_name(data, i)
        return j
    if c == ord('<'):
        if i + 1 < len(data) and data[i + 1] == ord('<'):
            _, j = read_dict(data, i)
            return j
        _, j = read_hex_string(data, i)
        return j
    if c == ord('('):
        _, j = read_literal_string(data, i)
        return j
    # generic token (number/keyword/etc.)
    _, j = read_word(data, i)
    return j


# ---------------- core rewrite ----------------

def remove_marked_content_wrappers(data: bytes) -> bytes:
    out = bytearray()
    i = 0
    n = len(data)

    while i < n:
        c = data[i]

        # Candidate start of BMC/BDC: must begin with a tag name
        if c == ord('/'):
            start = i
            _, j = read_name(data, i)
            k = skip_ws(data, j)

            # try BMC: /Tag BMC
            op, k2 = read_word(data, k)
            if op == b"BMC":
                # drop prefix "/Tag BMC", keep inner content
                i = k2
                continue

            # try BDC: /Tag <props> BDC   OR   /Tag /PropName BDC
            p = skip_single_operand(data, k)
            p_ws = skip_ws(data, p)
            op2, p2 = read_word(data, p_ws)
            if op2 == b"BDC":
                # drop prefix "/Tag <operand> BDC", keep inner content
                i = p2
                continue

            # not BMC/BDC start, copy original '/'
            out.append(data[start])
            i = start + 1
            continue

        # Remove EMC operators (standalone)
        if (c == ord('E') and data[i:i+3] == b"EMC"):
            prev_ok = (i == 0) or (data[i - 1] in WS + b"[]<>()/%")
            next_ok = (i + 3 >= n) or (data[i + 3] in WS + b"[]<>()/%")
            if prev_ok and next_ok:
                i += 3
                continue

        out.append(c)
        i += 1

    return bytes(out)


# ---------------- PDF traversal ----------------

def is_form_xobject(obj) -> bool:
    if not isinstance(obj, Stream):
        return False
    subtype = obj.get(Name("/Subtype"), None)
    typ = obj.get(Name("/Type"), None)
    return subtype == Name("/Form") or (typ == Name("/XObject") and subtype == Name("/Form"))


def page_content_bytes(page) -> bytes:
    c = page.get(Name("/Contents"), None)
    if c is None:
        return b""
    if isinstance(c, Stream):
        return c.read_bytes()
    if isinstance(c, Array):
        parts = []
        for s in c:
            if isinstance(s, Stream):
                parts.append(s.read_bytes())
        return b"\n".join(parts)
    return b""


def set_page_content_bytes(pdf: pikepdf.Pdf, page, data: bytes) -> None:
    page[Name("/Contents")] = pdf.make_stream(data)


def process_form_recursive(form: Stream, visited: Set[Tuple[int, int]]) -> int:
    changed = 0
    objgen = getattr(form, "objgen", None)
    if objgen and objgen in visited:
        return 0
    if objgen:
        visited.add(objgen)

    original = form.read_bytes()
    rewritten = remove_marked_content_wrappers(original)
    if rewritten != original:
        form.write(rewritten)
        changed += 1

    resources = form.get(Name("/Resources"), None)
    if isinstance(resources, Dictionary):
        changed += process_forms_in_resources(resources, visited)

    return changed


def process_forms_in_resources(resources: Dictionary, visited: Set[Tuple[int, int]]) -> int:
    changed = 0
    xobj = resources.get(Name("/XObject"), None)
    if not isinstance(xobj, Dictionary):
        return 0

    for _, xo in xobj.items():
        if is_form_xobject(xo):
            changed += process_form_recursive(xo, visited)

    return changed


def process_pdf(infile: str, outfile: str) -> None:
    with pikepdf.open(infile) as pdf:
        modified_streams = 0
        visited_forms: Set[Tuple[int, int]] = set()

        for page in pdf.pages:
            # page content
            original = page_content_bytes(page)
            if original:
                rewritten = remove_marked_content_wrappers(original)
                if rewritten != original:
                    set_page_content_bytes(pdf, page, rewritten)
                    modified_streams += 1

            # forms reachable from page resources
            resources = page.get(Name("/Resources"), None)
            if isinstance(resources, Dictionary):
                modified_streams += process_forms_in_resources(resources, visited_forms)

        pdf.save(outfile)

    print(f"Wrote {outfile} (modified streams: {modified_streams})")


def main():
    if len(sys.argv) != 3:
        print("Usage: python remove_marked_content_wrappers.py in.pdf out.pdf")
        sys.exit(2)

    process_pdf(sys.argv[1], sys.argv[2])


if __name__ == "__main__":
    main()