Let a project with no tracking in the door

analyze told agents to use audit_tracking when there was nothing to analyse,
and audit_tracking required a CSV — so the advice led to a closed door, and the
people who most need this tool were the ones who could not start. Its csv_path
is optional now. Without data it reads what the code logs and answers the
question that matters at that point: whether any of it could ever show that a
user got value. Logging only open_app and page_view is a finding, not an error.

It also returns how long to wait before analysing. Those numbers are not
invented — 7 days is how long a user must be watched before the code will call
them churned, 14 before regular and week-1 retention. Both were literals inside
classify_users and are now named constants that tracking_plan reads, so the
advice cannot drift from the rule it describes.

That closes the loop the tool was missing: add logging, come back in a week for
a first report, and the week after that history_compare has something to
compare against.

Author: 이혁진

analysis.py

@@ -17,6 +17,11 @@
 # classify_users와 find_aha_moments가 반드시 같은 기준을 쓴다 (리포트 안에서 숫자가 어긋나지 않게).
 REGULAR_DAYS_IN_2W = 10
 
+# 유저를 판정하려면 그만큼 지켜본 뒤여야 한다. 이 두 숫자가 곧
+# "로깅을 심었으면 언제 다시 오면 되는가"의 답이기도 하다 (tracking_plan이 그대로 쓴다).
+CHURN_JUDGEMENT_DAYS = 7    # 이만큼 지나야 '한 번 쓰고 떠났다'고 말할 수 있다
+REGULAR_JUDGEMENT_DAYS = 14  # 이만큼 지나야 단골 판정과 1주차 리텐션이 나온다
+
 # 허영 지표 — 가치를 증명하지 않는 이벤트. value_event로 못 쓰게 막고,
 # 아하 후보에서도 뺀다. 이름만 다른 같은 개념들을 모아둔다.
 VANITY_EVENTS = {
@@ -292,11 +297,11 @@ def classify_users(users: dict[str, UserRow], today: date) -> dict[str, list[str
         active = sorted(u.value_days)
         tenure = (today - u.signup).days + 1
         recent2w = [d for d in active if (today - d).days < 14]
-        if len(active) <= 1 and tenure >= 7:
+        if len(active) <= 1 and tenure >= CHURN_JUDGEMENT_DAYS:
             buckets["churned"].append(u.user_id)
         elif active and all(d.weekday() >= 5 for d in active) and len(active) >= 2:
             buckets["weekend_only"].append(u.user_id)
-        elif tenure >= 14 and len(recent2w) >= REGULAR_DAYS_IN_2W:
+        elif tenure >= REGULAR_JUDGEMENT_DAYS and len(recent2w) >= REGULAR_DAYS_IN_2W:
             buckets["regular"].append(u.user_id)
         else:
             buckets["casual"].append(u.user_id)
@@ -391,21 +396,54 @@ def top_aha(results: list[dict]) -> dict | None:
     )
 
 
-def audit_tracking(code_events: list[str], data_events: set[str]) -> dict:
-    """코드에서 찾은 이벤트 목록과 실제 데이터에 찍힌 이벤트를 대조한다.
+def audit_tracking(code_events: list[str], data_events: set[str] | None = None) -> dict:
+    """코드에서 찾은 이벤트를 점검한다. 데이터가 있으면 대조까지 한다.
+
+    data_events가 없어도 답할 게 있다 — 오히려 이 경우가 이 툴이 제일 필요한
+    상황이다(추적을 안 해서 데이터가 없는 것). 코드만 봐도 "심어둔 게 전부
+    허영 지표라 '유저가 가치를 얻었나'에 영원히 답할 수 없다"는 진단이 나온다.
 
+    데이터가 있으면 추가로:
     - 코드에만 있음 → 심어놨는데 한 번도 안 찍힘 (고장이거나, 아무도 안 쓰는 기능)
     - 데이터에만 있음 → 코드에서 못 찾음 (죽은 코드거나, 스캔 누락)
-    - 양쪽 다 있음 → 정상 추적 중
     """
     code = set(code_events)
-    return {
+    usable = sorted(e for e in code if not is_vanity(e))
+    out = {
+        "in_code": sorted(code),
+        "usable_in_code": usable,
+        "vanity_in_code": sorted(e for e in code if is_vanity(e)),
+        "can_measure_value": bool(usable),
+    }
+    if data_events is None:
+        return out
+    return out | {
         "tracked_ok": sorted(code & data_events),
         "in_code_never_fired": sorted(code - data_events),
         "in_data_not_in_code": sorted(data_events - code),
     }
 
 
+def tracking_plan(code_events: list[str]) -> dict:
+    """추적이 부족한 사람에게 줄 답: 지금 뭐가 부족하고, 언제 다시 오면 되는가.
+
+    '며칠 뒤'는 지어낸 숫자가 아니라 판정 기준에서 그대로 나온다 —
+    이탈은 CHURN_JUDGEMENT_DAYS, 단골과 1주차 리텐션은 REGULAR_JUDGEMENT_DAYS.
+    """
+    audit = audit_tracking(code_events)
+    return audit | {
+        "come_back_in_days": {
+            "first_signal": CHURN_JUDGEMENT_DAYS,
+            "full_picture": REGULAR_JUDGEMENT_DAYS,
+        },
+        "why_those_days": (
+            f"Calling a user churned needs {CHURN_JUDGEMENT_DAYS} days of watching them; "
+            f"calling one a regular, and week-1 retention, needs {REGULAR_JUDGEMENT_DAYS}. "
+            "Before that the code has nothing honest to say."
+        ),
+    }
+
+
 def onboarding_funnel(users: dict[str, UserRow], today: date) -> dict:
     """온보딩 퍼널: 새 유저가 어느 계단에서 떨어지는지.
 

server.py

@@ -81,9 +81,12 @@ def analyze(
                 "SELECT user_id, created_at::date AS date, 'purchase' AS event FROM orders "
                 "UNION ALL SELECT user_id, added_at::date, 'add_to_wishlist' FROM wishlist_items",
                 "4. Call analyze again with the CSV it wrote.",
-                "If there is no database and nothing is tracked, say so plainly — this "
-                "tool cannot help yet. Offer to add event logging instead: find the "
-                "handlers for the product's core actions and use audit_tracking.",
+                "If there is no database and nothing is tracked, say so plainly — "
+                "this tool cannot report on a product that records nothing. Then do "
+                "the useful thing instead: read the code for the core actions and "
+                "call audit_tracking with them (no csv_path needed). It names the "
+                "holes and tells you how many days of data are needed before "
+                "analyze can say anything honest.",
             ],
         }
 
@@ -99,9 +102,10 @@ def analyze(
                 "what_to_do": (
                     "Nothing in this data looks like a value event: every action is "
                     "either a vanity metric, done by too few users, or never repeated. "
-                    "That is a tracking problem, not a product problem. Tell the user "
-                    "which core actions are not being recorded and offer to add logging "
-                    "(audit_tracking has the procedure)."
+                    "That is a tracking problem, not a product problem. Read the code "
+                    "for the product's core actions and call audit_tracking with what "
+                    "you find — it works without data and comes back with the holes, "
+                    "and with how long to wait once logging is added."
                 ),
             }
         value_event = best["event"]
@@ -239,11 +243,12 @@ def find_aha_moments(csv_path: str, value_event: str) -> dict:
 
 
 @mcp.tool()
-def audit_tracking(code_events: list[str], csv_path: str) -> dict:
-    """Compare the events logged in the code against the events in the data.
+def audit_tracking(code_events: list[str], csv_path: str | None = None) -> dict:
+    """Check what the code logs, and — if there is data — whether it arrives.
 
-    This is also the tool for a project that tracks nothing yet — there is no
-    data to analyse, but there is code to read.
+    Use this when analyze says there is nothing to work with. A project that
+    tracks nothing has no data to analyse, but it has code to read, so leave
+    csv_path out and this still answers.
 
     How to use it:
     1. Search the codebase for logging calls yourself. Common shapes:
@@ -260,10 +265,21 @@ def audit_tracking(code_events: list[str], csv_path: str) -> dict:
        that belongs in that file, in that function, matching the surrounding
        style, show it, and ask whether to add it. Follow the project's existing
        naming (follow_artist if it is snake_case, followArtist if camelCase).
+    6. Once logging is in, tell them when to come back. come_back_in_days is in
+       the result and is not a guess — it is how long the code must watch a user
+       before it can honestly call them churned or a regular. Saying "run this
+       again tomorrow" would produce a report with nothing in it.
     """
+    if csv_path is None:
+        return analysis.tracking_plan(code_events)
+
     events = analysis.load_events(csv_path)
     data_events = {e["event"] for e in events}
-    return analysis.audit_tracking(code_events, data_events)
+    result = analysis.audit_tracking(code_events, data_events)
+    if not result["usable_in_code"]:
+        # 데이터가 있어도 전부 허영 지표면 '가치를 얻었나'에 답할 수 없다
+        result |= analysis.tracking_plan(code_events)
+    return result
 
 
 @mcp.tool()

tests/test_pipeline.py

@@ -299,3 +299,50 @@ def test_unmatched_columns_say_what_to_do(tmp_path):
     with pytest.raises(analysis.EventFileError) as e:
         analysis.load_events(path)
     assert "a" in str(e.value) and "AS date" in str(e.value)
+
+
+# ── 추적이 없는 사람도 들어올 수 있는가 ──────────────────────────────────
+
+def test_audit_tracking_works_without_any_data():
+    """추적을 안 해서 온 사람인데 데이터를 요구하면 안 된다 — 이 툴이 제일
+    필요한 상황이 바로 데이터가 없는 상황이다."""
+    import server
+    r = server.audit_tracking(["open_app", "page_view"])
+    assert r["can_measure_value"] is False
+    assert r["vanity_in_code"] == ["open_app", "page_view"]
+    assert r["come_back_in_days"]["first_signal"] == analysis.CHURN_JUDGEMENT_DAYS
+    assert r["come_back_in_days"]["full_picture"] == analysis.REGULAR_JUDGEMENT_DAYS
+
+
+def test_audit_tracking_still_compares_when_data_exists(tmp_path):
+    import server
+    path = write_csv(tmp_path / "e.csv", events_for(6, 10), ["user_id", "date", "event"])
+    r = server.audit_tracking(["purchase", "share"], path)
+    assert r["tracked_ok"] == ["purchase"]
+    assert r["in_code_never_fired"] == ["share"]
+
+
+def test_come_back_days_come_from_the_judgement_thresholds():
+    """'며칠 뒤 오세요'가 지어낸 숫자면 안 된다. 유저를 그만큼 지켜봐야
+    이탈·단골을 판정할 수 있다는 코드의 기준에서 나와야 한다."""
+    plan = analysis.tracking_plan(["purchase"])
+    assert plan["come_back_in_days"] == {
+        "first_signal": analysis.CHURN_JUDGEMENT_DAYS,
+        "full_picture": analysis.REGULAR_JUDGEMENT_DAYS,
+    }
+    assert plan["can_measure_value"] is True
+
+
+def test_analyze_sends_a_trackless_project_somewhere_reachable(tmp_path):
+    """막다른 길로 안내하면 안 된다 — analyze가 가리키는 곳이 실제로 열려 있어야."""
+    import server
+    for guidance in (
+        " ".join(server.analyze()["what_to_do"]),
+        server.analyze(
+            write_csv(tmp_path / "e.csv", events_for(9, 20, event="page_view"),
+                      ["user_id", "date", "event"]),
+            output_path=str(tmp_path / "r.html"),
+        )["what_to_do"],
+    ):
+        assert "audit_tracking" in guidance
+    server.audit_tracking(["page_view"])   # 안내받은 대로 부르면 열려야 한다