fix: expose session lineage metadata in API (#1370)

PR #1358 added the client-side lineage collapse helper, but
/api/sessions often did not include _lineage_root_id for the WebUI
JSON sessions visible in the sidebar. In that case the helper has no
grouping key and multiple same-title continuation rows remain visible.

This PR:
- Reads parent_session_id and end_reason from state.db.sessions for
  the WebUI sidebar's session ids
- Walks the parent chain when end_reason is 'compression' or
  'cli_close', producing _lineage_root_id and _compression_segment_count
- Cycle-detects via a 'seen' set
- Preserves projected lineage metadata on imported/gateway session rows
- Allows sidebar collapse to group cross-surface continuation chains
  (CLI-close → WebUI continuation) while keeping non-continuation
  parent rows flat

Co-authored-by: Dennis Soong <dso2ng@gmail.com>

api/agent_sessions.py

@@ -255,3 +255,68 @@ def read_importable_agent_session_rows(
         if limit is None:
             return projected
         return projected[:max(0, int(limit))]
+
+
+
+def read_session_lineage_metadata(db_path: Path, session_ids: list[str] | set[str]) -> dict[str, dict]:
+    """Return compression-lineage metadata for known WebUI sidebar sessions.
+
+    WebUI sessions are persisted as JSON files, but Hermes Agent also mirrors
+    them into ``state.db.sessions`` for insights/session history. Compression
+    and cross-surface continuation create parent chains there. ``/api/sessions``
+    needs to surface that lineage to the sidebar so client-side collapse can
+    group logical continuations without mutating or deleting any session files.
+
+    Missing DBs, old schemas, or incomplete rows degrade to an empty mapping.
+    """
+    wanted = {str(sid) for sid in (session_ids or []) if sid}
+    db_path = Path(db_path)
+    if not wanted or not db_path.exists():
+        return {}
+
+    try:
+        with sqlite3.connect(str(db_path)) as conn:
+            conn.row_factory = sqlite3.Row
+            cur = conn.cursor()
+            cur.execute("PRAGMA table_info(sessions)")
+            session_cols = {row[1] for row in cur.fetchall()}
+            if 'parent_session_id' not in session_cols or 'end_reason' not in session_cols:
+                return {}
+            cur.execute("SELECT id, parent_session_id, end_reason FROM sessions")
+            rows = {row['id']: dict(row) for row in cur.fetchall()}
+    except Exception:
+        return {}
+
+    metadata: dict[str, dict] = {}
+    for sid in wanted:
+        row = rows.get(sid)
+        if not row:
+            continue
+
+        parent_id = row.get('parent_session_id')
+        if parent_id:
+            metadata.setdefault(sid, {})['parent_session_id'] = parent_id
+
+        root_id = sid
+        current_id = sid
+        segment_count = 1
+        seen = {sid}
+        while True:
+            current = rows.get(current_id)
+            parent_id = current.get('parent_session_id') if current else None
+            parent = rows.get(parent_id) if parent_id else None
+            if not parent or parent_id in seen:
+                break
+            if parent.get('end_reason') not in {'compression', 'cli_close'}:
+                break
+            root_id = parent_id
+            current_id = parent_id
+            seen.add(parent_id)
+            segment_count += 1
+
+        if root_id != sid:
+            entry = metadata.setdefault(sid, {})
+            entry['_lineage_root_id'] = root_id
+            entry['_compression_segment_count'] = segment_count
+
+    return metadata

api/models.py

@@ -15,7 +15,7 @@ from api.config import (
     get_effective_default_model, _get_session_agent_lock,
 )
 from api.workspace import get_last_workspace
-from api.agent_sessions import read_importable_agent_session_rows
+from api.agent_sessions import read_importable_agent_session_rows, read_session_lineage_metadata
 
 logger = logging.getLogger(__name__)
 
@@ -746,6 +746,31 @@ def _hide_from_default_sidebar(session: dict) -> bool:
     return source == 'cron' or sid.startswith('cron_')
 
 
+def _active_state_db_path() -> Path:
+    """Return state.db for the active Hermes profile, degrading to HERMES_HOME."""
+    try:
+        from api.profiles import get_active_hermes_home
+        hermes_home = Path(get_active_hermes_home()).expanduser().resolve()
+    except Exception:
+        hermes_home = Path(os.getenv('HERMES_HOME', str(HOME / '.hermes'))).expanduser().resolve()
+    return hermes_home / 'state.db'
+
+
+def _enrich_sidebar_lineage_metadata(sessions: list[dict]) -> None:
+    """Attach state.db compression lineage metadata used by sidebar collapse."""
+    try:
+        metadata = read_session_lineage_metadata(
+            _active_state_db_path(),
+            {s.get('session_id') for s in sessions},
+        )
+    except Exception:
+        return
+    for session in sessions:
+        sid = session.get('session_id')
+        if sid in metadata:
+            session.update(metadata[sid])
+
+
 def all_sessions():
     active_stream_ids = _active_stream_ids()
     # Phase C: try index first for O(1) read; fall back to full scan
@@ -804,6 +829,7 @@ def all_sessions():
             for s in result:
                 if not s.get('profile'):
                     s['profile'] = 'default'
+            _enrich_sidebar_lineage_metadata(result)
             return result
         except Exception:
             logger.debug("Failed to load session index, falling back to full scan")
@@ -832,6 +858,7 @@ def all_sessions():
     for s in result:
         if not s.get('profile'):
             s['profile'] = 'default'
+    _enrich_sidebar_lineage_metadata(result)
     return result
 
 
@@ -1015,6 +1042,10 @@ def get_cli_sessions() -> list:
                 'raw_source': row.get('raw_source'),
                 'session_source': row.get('session_source'),
                 'source_label': row.get('source_label'),
+                'parent_session_id': row.get('parent_session_id'),
+                '_lineage_root_id': row.get('_lineage_root_id'),
+                '_lineage_tip_id': row.get('_lineage_tip_id'),
+                '_compression_segment_count': row.get('_compression_segment_count'),
                 'is_cli_session': True,
             })
     except Exception as _cli_err:

tests/test_gateway_sync.py

@@ -333,6 +333,9 @@ def test_compression_chain_collapses_to_latest_tip_in_sidebar():
         # bubbles to the top by true recency, not by the root's stale activity.
         # tip messages are at t0+201 and t0+202, so last_activity = t0 + 202.
         assert abs(tip.get('updated_at') - (t0 + 202)) < 0.01
+        assert tip.get('_lineage_root_id') == 'chain_root_001'
+        assert tip.get('_lineage_tip_id') == 'chain_tip_001'
+        assert tip.get('_compression_segment_count') == 3
 
         from api.agent_sessions import read_importable_agent_session_rows
 

tests/test_session_lineage_metadata_api.py

@@ -0,0 +1,156 @@
+"""Regression tests for /api/sessions lineage metadata used by sidebar collapse."""
+
+import sqlite3
+import time
+
+import pytest
+
+import api.models as models
+from api.models import SESSIONS, STREAMS, Session, all_sessions
+
+
+@pytest.fixture(autouse=True)
+def _isolate(tmp_path, monkeypatch):
+    session_dir = tmp_path / "sessions"
+    session_dir.mkdir()
+    index_file = session_dir / "_index.json"
+    state_db = tmp_path / "state.db"
+    monkeypatch.setattr(models, "SESSION_DIR", session_dir)
+    monkeypatch.setattr(models, "SESSION_INDEX_FILE", index_file)
+    monkeypatch.setattr(models, "_active_state_db_path", lambda: state_db)
+    SESSIONS.clear()
+    STREAMS.clear()
+    yield state_db
+    SESSIONS.clear()
+    STREAMS.clear()
+
+
+def _ensure_state_db(path):
+    conn = sqlite3.connect(str(path))
+    conn.executescript(
+        """
+        CREATE TABLE sessions (
+            id TEXT PRIMARY KEY,
+            source TEXT,
+            title TEXT,
+            model TEXT,
+            started_at REAL NOT NULL,
+            message_count INTEGER DEFAULT 0,
+            parent_session_id TEXT,
+            ended_at REAL,
+            end_reason TEXT
+        );
+        """
+    )
+    return conn
+
+
+def _insert_state_row(conn, sid, *, parent=None, ended_at=None, end_reason=None, started_at=None):
+    conn.execute(
+        """
+        INSERT INTO sessions
+        (id, source, title, model, started_at, message_count, parent_session_id, ended_at, end_reason)
+        VALUES (?, 'webui', ?, 'openai/gpt-5', ?, 2, ?, ?, ?)
+        """,
+        (sid, sid, started_at or time.time(), parent, ended_at, end_reason),
+    )
+    conn.commit()
+
+
+def _save_webui_session(sid, *, title, updated_at):
+    session = Session(
+        session_id=sid,
+        title=title,
+        messages=[{"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi"}],
+        updated_at=updated_at,
+    )
+    session.save(touch_updated_at=False)
+    return session
+
+
+def test_all_sessions_exposes_state_db_lineage_metadata_for_webui_json_sessions(_isolate):
+    """PR #1358 can only collapse rows when /api/sessions exposes lineage keys."""
+    conn = _ensure_state_db(_isolate)
+    t0 = time.time() - 100
+    try:
+        _save_webui_session("lineage_api_root", title="Hermes WebUI", updated_at=t0)
+        _save_webui_session("lineage_api_tip", title="Hermes WebUI #2", updated_at=t0 + 10)
+        _insert_state_row(
+            conn,
+            "lineage_api_root",
+            started_at=t0,
+            ended_at=t0 + 5,
+            end_reason="compression",
+        )
+        _insert_state_row(
+            conn,
+            "lineage_api_tip",
+            parent="lineage_api_root",
+            started_at=t0 + 6,
+        )
+
+        rows = {row["session_id"]: row for row in all_sessions()}
+
+        assert rows["lineage_api_tip"].get("parent_session_id") == "lineage_api_root"
+        assert rows["lineage_api_tip"].get("_lineage_root_id") == "lineage_api_root"
+        assert rows["lineage_api_tip"].get("_compression_segment_count") == 2
+        assert "_lineage_root_id" not in rows["lineage_api_root"]
+    finally:
+        conn.close()
+
+
+def test_non_compression_state_db_parent_does_not_create_sidebar_lineage(_isolate):
+    conn = _ensure_state_db(_isolate)
+    t0 = time.time() - 100
+    try:
+        _save_webui_session("lineage_api_plain_parent", title="Parent", updated_at=t0)
+        _save_webui_session("lineage_api_plain_child", title="Child", updated_at=t0 + 10)
+        _insert_state_row(
+            conn,
+            "lineage_api_plain_parent",
+            started_at=t0,
+            ended_at=t0 + 5,
+            end_reason="user_stop",
+        )
+        _insert_state_row(
+            conn,
+            "lineage_api_plain_child",
+            parent="lineage_api_plain_parent",
+            started_at=t0 + 6,
+        )
+
+        rows = {row["session_id"]: row for row in all_sessions()}
+
+        assert rows["lineage_api_plain_child"].get("parent_session_id") == "lineage_api_plain_parent"
+        assert "_lineage_root_id" not in rows["lineage_api_plain_child"]
+    finally:
+        conn.close()
+
+
+
+def test_cli_close_parent_preserves_cross_surface_continuation_lineage(_isolate):
+    conn = _ensure_state_db(_isolate)
+    t0 = time.time() - 100
+    try:
+        _save_webui_session("lineage_api_cli_parent", title="Hermes WebUI #8", updated_at=t0)
+        _save_webui_session("lineage_api_webui_child", title="Hermes WebUI #8", updated_at=t0 + 10)
+        _insert_state_row(
+            conn,
+            "lineage_api_cli_parent",
+            started_at=t0,
+            ended_at=t0 + 5,
+            end_reason="cli_close",
+        )
+        _insert_state_row(
+            conn,
+            "lineage_api_webui_child",
+            parent="lineage_api_cli_parent",
+            started_at=t0 + 6,
+        )
+
+        rows = {row["session_id"]: row for row in all_sessions()}
+
+        assert rows["lineage_api_webui_child"].get("parent_session_id") == "lineage_api_cli_parent"
+        assert rows["lineage_api_webui_child"].get("_lineage_root_id") == "lineage_api_cli_parent"
+    finally:
+        conn.close()