|
| 1 | +import os |
| 2 | +import re |
| 3 | +import sys |
| 4 | +import requests |
| 5 | +from urllib.parse import urlparse |
| 6 | +from datetime import datetime, timezone, timedelta |
| 7 | + |
| 8 | +SAVE_DIR = "docs/contribute" |
| 9 | +STATIC_IMG_DIR = "static/img/contribute" |
| 10 | +OVERVIEW_FILE = f"{SAVE_DIR}/overview.md" |
| 11 | +SKIP_FILES = {"overview.md", "TEMPLATE.md"} |
| 12 | + |
| 13 | +NOTION_PROPERTY_TITLE = os.environ.get("NOTION_PROPERTY_TITLE", "제목") |
| 14 | +NOTION_PROPERTY_DATE = os.environ.get("NOTION_PROPERTY_DATE", "날짜") |
| 15 | +NOTION_PROPERTY_PROJECT = os.environ.get("NOTION_PROPERTY_PROJECT", "프로젝트") |
| 16 | +NOTION_PROPERTY_TYPE = os.environ.get("NOTION_PROPERTY_TYPE", "유형") |
| 17 | +NOTION_PROPERTY_STATUS = os.environ.get("NOTION_PROPERTY_STATUS", "상태") |
| 18 | +NOTION_PROPERTY_URL = os.environ.get("NOTION_PROPERTY_URL", "URL") |
| 19 | + |
| 20 | + |
| 21 | +def normalize_notion_database_id(raw): |
| 22 | + raw = (raw or "").strip() |
| 23 | + if not raw: |
| 24 | + return raw |
| 25 | + if raw.lower().startswith(("http://", "https://")): |
| 26 | + from urllib.parse import urlparse as _up |
| 27 | + path = _up(raw).path.strip("/") |
| 28 | + blob = "/".join(path.split("/")[-2:]) if path else "" |
| 29 | + else: |
| 30 | + blob = raw.split("?")[0] |
| 31 | + found = re.findall(r"[0-9a-fA-F]{32}", blob.replace("-", "")) |
| 32 | + if not found: |
| 33 | + compact = re.sub(r"[^0-9a-fA-F]", "", blob) |
| 34 | + if len(compact) >= 32: |
| 35 | + found = [compact[:32]] |
| 36 | + if not found: |
| 37 | + return raw |
| 38 | + h = found[0].lower() |
| 39 | + return f"{h[:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:]}" |
| 40 | + |
| 41 | + |
| 42 | +NOTION_TOKEN = os.environ["NOTION_TOKEN"] |
| 43 | +DATABASE_ID = normalize_notion_database_id(os.environ["NOTION_CONTRIBUTE_DATABASE_ID"]) |
| 44 | + |
| 45 | +headers = { |
| 46 | + "Authorization": f"Bearer {NOTION_TOKEN}", |
| 47 | + "Content-Type": "application/json", |
| 48 | + "Notion-Version": "2022-06-28", |
| 49 | +} |
| 50 | + |
| 51 | + |
| 52 | +def verify_database_access(): |
| 53 | + url = f"https://api.notion.com/v1/databases/{DATABASE_ID}" |
| 54 | + r = requests.get(url, headers=headers) |
| 55 | + data = r.json() |
| 56 | + print(f">> GET /databases/{{id}} → HTTP {r.status_code}") |
| 57 | + if r.status_code != 200 or data.get("object") == "error": |
| 58 | + print(f">> ERROR: {data.get('message', data)}") |
| 59 | + return False |
| 60 | + names = list(data.get("properties", {}).keys()) |
| 61 | + print(f">> DB 속성 ({len(names)}개): {names}") |
| 62 | + return True |
| 63 | + |
| 64 | + |
| 65 | +def read_select(props, prop_name): |
| 66 | + p = props.get(prop_name, {}) |
| 67 | + ptype = p.get("type") |
| 68 | + if ptype == "select": |
| 69 | + s = p.get("select") |
| 70 | + return s["name"] if s else "" |
| 71 | + return "" |
| 72 | + |
| 73 | + |
| 74 | +def read_url(props, prop_name): |
| 75 | + p = props.get(prop_name, {}) |
| 76 | + return p.get("url") or "" |
| 77 | + |
| 78 | + |
| 79 | +def read_title_plain(props, prop_name): |
| 80 | + p = props.get(prop_name, {}) |
| 81 | + if p.get("type") != "title": |
| 82 | + return None |
| 83 | + inner = p.get("title", []) |
| 84 | + try: |
| 85 | + return inner[0]["plain_text"] |
| 86 | + except (IndexError, KeyError, TypeError): |
| 87 | + return None |
| 88 | + |
| 89 | + |
| 90 | +def read_date_start(props, prop_name): |
| 91 | + p = props.get(prop_name, {}) |
| 92 | + if p.get("type") != "date": |
| 93 | + return None |
| 94 | + inner = p.get("date") |
| 95 | + if not inner: |
| 96 | + return None |
| 97 | + start = inner.get("start", "") |
| 98 | + return start[:10] if len(start) >= 10 else None |
| 99 | + |
| 100 | + |
| 101 | +def get_page_blocks(page_id): |
| 102 | + url = f"https://api.notion.com/v1/blocks/{page_id}/children" |
| 103 | + return requests.get(url, headers=headers).json().get("results", []) |
| 104 | + |
| 105 | + |
| 106 | +def extract_text(rich_text_list): |
| 107 | + parts = [] |
| 108 | + for text in rich_text_list: |
| 109 | + plain = text["plain_text"] |
| 110 | + href = text.get("href") |
| 111 | + parts.append(f"[{plain}]({href})" if href else plain) |
| 112 | + return " ".join(parts) |
| 113 | + |
| 114 | + |
| 115 | +def download_image(url, slug, index): |
| 116 | + save_dir = f"{STATIC_IMG_DIR}/{slug}" |
| 117 | + os.makedirs(save_dir, exist_ok=True) |
| 118 | + parsed = urlparse(url) |
| 119 | + ext = os.path.splitext(parsed.path)[1].lower() |
| 120 | + if ext not in (".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg"): |
| 121 | + ext = ".jpg" |
| 122 | + filename = f"img-{index:02d}{ext}" |
| 123 | + filepath = f"{save_dir}/{filename}" |
| 124 | + if not os.path.exists(filepath): |
| 125 | + r = requests.get(url, timeout=30) |
| 126 | + r.raise_for_status() |
| 127 | + with open(filepath, "wb") as f: |
| 128 | + f.write(r.content) |
| 129 | + print(f">> 이미지 저장: {filepath}") |
| 130 | + return f"/img/contribute/{slug}/{filename}" |
| 131 | + |
| 132 | + |
| 133 | +def blocks_to_markdown(blocks, slug): |
| 134 | + image_counter = [0] |
| 135 | + parts = [] |
| 136 | + for block in blocks: |
| 137 | + b_type = block["type"] |
| 138 | + if b_type in ("paragraph", "heading_1", "heading_2", "heading_3", |
| 139 | + "bulleted_list_item", "numbered_list_item", "quote", "callout"): |
| 140 | + content = extract_text(block[b_type].get("rich_text", [])) |
| 141 | + if not content: |
| 142 | + continue |
| 143 | + if b_type == "paragraph": |
| 144 | + parts.append(content + "\n\n") |
| 145 | + elif b_type == "heading_1": |
| 146 | + parts.append(f"## {content}\n\n") |
| 147 | + elif b_type == "heading_2": |
| 148 | + parts.append(f"## {content}\n\n") |
| 149 | + elif b_type == "heading_3": |
| 150 | + parts.append(f"### {content}\n\n") |
| 151 | + elif b_type == "bulleted_list_item": |
| 152 | + parts.append(f"- {content}\n") |
| 153 | + elif b_type == "numbered_list_item": |
| 154 | + parts.append(f"1. {content}\n") |
| 155 | + elif b_type in ("quote", "callout"): |
| 156 | + parts.append(f"> {content}\n\n") |
| 157 | + elif b_type == "code": |
| 158 | + lang = block["code"].get("language", "text") |
| 159 | + content = extract_text(block["code"].get("rich_text", [])) |
| 160 | + parts.append(f"```{lang}\n{content}\n```\n\n") |
| 161 | + elif b_type == "image": |
| 162 | + url = (block["image"].get("file", {}).get("url") |
| 163 | + or block["image"].get("external", {}).get("url") or "") |
| 164 | + if url: |
| 165 | + try: |
| 166 | + local = download_image(url, slug, image_counter[0]) |
| 167 | + image_counter[0] += 1 |
| 168 | + parts.append(f"\n\n") |
| 169 | + except Exception as e: |
| 170 | + print(f">> WARN: 이미지 다운로드 실패: {e}") |
| 171 | + elif b_type == "divider": |
| 172 | + parts.append("---\n\n") |
| 173 | + return "".join(parts) |
| 174 | + |
| 175 | + |
| 176 | +def slugify(text): |
| 177 | + slug = re.sub(r"[^\w\s-]", "", text.lower()) |
| 178 | + slug = re.sub(r"[\s_]+", "-", slug).strip("-") |
| 179 | + return slug or "contribution" |
| 180 | + |
| 181 | + |
| 182 | +SYNC_MAP_FILE = f"{SAVE_DIR}/.contribute-sync.json" |
| 183 | + |
| 184 | + |
| 185 | +def load_sync_map(): |
| 186 | + if not os.path.exists(SYNC_MAP_FILE): |
| 187 | + return {} |
| 188 | + import json |
| 189 | + with open(SYNC_MAP_FILE, encoding="utf-8") as f: |
| 190 | + return json.load(f) |
| 191 | + |
| 192 | + |
| 193 | +def save_sync_map(mapping): |
| 194 | + import json |
| 195 | + with open(SYNC_MAP_FILE, "w", encoding="utf-8") as f: |
| 196 | + json.dump(mapping, f, ensure_ascii=False, indent=2) |
| 197 | + |
| 198 | + |
| 199 | +def update_overview_count(count): |
| 200 | + """overview.md의 '현재: X건' 숫자를 갱신.""" |
| 201 | + if not os.path.exists(OVERVIEW_FILE): |
| 202 | + return |
| 203 | + with open(OVERVIEW_FILE, encoding="utf-8") as f: |
| 204 | + content = f.read() |
| 205 | + updated = re.sub(r"현재: \d+건", f"현재: {count}건", content) |
| 206 | + if updated != content: |
| 207 | + with open(OVERVIEW_FILE, "w", encoding="utf-8") as f: |
| 208 | + f.write(updated) |
| 209 | + print(f">> overview.md 카운트 갱신: {count}건") |
| 210 | + |
| 211 | + |
| 212 | +def save_contribution(page, date_str, existing_map): |
| 213 | + if len(date_str) > 10: |
| 214 | + date_str = date_str[:10] |
| 215 | + |
| 216 | + page_id = page["id"] |
| 217 | + props = page.get("properties", {}) |
| 218 | + |
| 219 | + title = read_title_plain(props, NOTION_PROPERTY_TITLE) or "제목없음" |
| 220 | + project = read_select(props, NOTION_PROPERTY_PROJECT) |
| 221 | + contrib_type = read_select(props, NOTION_PROPERTY_TYPE) |
| 222 | + status = read_select(props, NOTION_PROPERTY_STATUS) |
| 223 | + upstream_url = read_url(props, NOTION_PROPERTY_URL) |
| 224 | + |
| 225 | + slug = slugify(f"{project}-{title}" if project else title) |
| 226 | + new_filename = f"{SAVE_DIR}/{date_str}-{slug}.md" |
| 227 | + |
| 228 | + # 파일명 변경 처리 |
| 229 | + old_filename = existing_map.get(page_id) |
| 230 | + if old_filename and old_filename != new_filename and os.path.exists(old_filename): |
| 231 | + os.remove(old_filename) |
| 232 | + print(f">> 이름 변경으로 기존 파일 삭제: {old_filename}") |
| 233 | + |
| 234 | + # 기여 정보 표 생성 |
| 235 | + display_title = f"[{project}] {title}" if project else title |
| 236 | + url_cell = f"[링크]({upstream_url})" if upstream_url else "—" |
| 237 | + |
| 238 | + info_table = ( |
| 239 | + "## 기여 정보\n\n" |
| 240 | + "| 항목 | 내용 |\n" |
| 241 | + "|------|------|\n" |
| 242 | + f"| 프로젝트 | {project or '—'} |\n" |
| 243 | + f"| 유형 | {contrib_type or '—'} |\n" |
| 244 | + f"| 날짜 | {date_str} |\n" |
| 245 | + f"| 상태 | `{status}` |\n" |
| 246 | + f"| 링크 | {url_cell} |\n\n" |
| 247 | + ) |
| 248 | + |
| 249 | + blocks = get_page_blocks(page_id) |
| 250 | + body = blocks_to_markdown(blocks, slug) |
| 251 | + |
| 252 | + safe_title = display_title.replace('"', '\\"') |
| 253 | + frontmatter = ( |
| 254 | + f"---\n" |
| 255 | + f"id: {slug}\n" |
| 256 | + f'title: "{safe_title}"\n' |
| 257 | + f"slug: /contribute/{slug}\n" |
| 258 | + f"description: \"{title} ({date_str}, {status})\"\n" |
| 259 | + f"---\n\n" |
| 260 | + ) |
| 261 | + |
| 262 | + os.makedirs(SAVE_DIR, exist_ok=True) |
| 263 | + with open(new_filename, "w", encoding="utf-8") as f: |
| 264 | + f.write(frontmatter + info_table + body) |
| 265 | + |
| 266 | + return title, new_filename |
| 267 | + |
| 268 | + |
| 269 | +def remove_orphans(synced_files): |
| 270 | + if not os.path.isdir(SAVE_DIR): |
| 271 | + return |
| 272 | + for fname in os.listdir(SAVE_DIR): |
| 273 | + if fname in SKIP_FILES or not fname.endswith(".md"): |
| 274 | + continue |
| 275 | + fpath = os.path.join(SAVE_DIR, fname) |
| 276 | + if fpath not in synced_files: |
| 277 | + os.remove(fpath) |
| 278 | + print(f">> 미추적 파일 삭제: {fpath}") |
| 279 | + |
| 280 | + |
| 281 | +def main(): |
| 282 | + if not verify_database_access(): |
| 283 | + sys.exit(1) |
| 284 | + |
| 285 | + fetch_mode = os.environ.get("FETCH_MODE", "ALL") # 기여는 기본 ALL |
| 286 | + url = f"https://api.notion.com/v1/databases/{DATABASE_ID}/query" |
| 287 | + payload = {} |
| 288 | + |
| 289 | + if fetch_mode == "DAILY": |
| 290 | + kst = timezone(timedelta(hours=9)) |
| 291 | + target_date = (datetime.now(kst) - timedelta(days=1)).strftime("%Y-%m-%d") |
| 292 | + print(f">> [모드: 일간] {target_date} 조회") |
| 293 | + payload["filter"] = { |
| 294 | + "property": NOTION_PROPERTY_DATE, |
| 295 | + "date": {"equals": target_date}, |
| 296 | + } |
| 297 | + else: |
| 298 | + print(">> [모드: 전체] 모든 기여 조회") |
| 299 | + |
| 300 | + existing_map = load_sync_map() |
| 301 | + has_more = True |
| 302 | + next_cursor = None |
| 303 | + saved = 0 |
| 304 | + synced_files = set() |
| 305 | + |
| 306 | + while has_more: |
| 307 | + if next_cursor: |
| 308 | + payload["start_cursor"] = next_cursor |
| 309 | + res = requests.post(url, headers=headers, json=payload) |
| 310 | + data = res.json() |
| 311 | + if res.status_code != 200 or data.get("object") == "error": |
| 312 | + print(f">> ERROR: {data.get('message', data)}") |
| 313 | + sys.exit(1) |
| 314 | + |
| 315 | + pages = data.get("results", []) |
| 316 | + print(f">> 페이지 수: {len(pages)}") |
| 317 | + |
| 318 | + for page in pages: |
| 319 | + props = page.get("properties", {}) |
| 320 | + page_date = read_date_start(props, NOTION_PROPERTY_DATE) |
| 321 | + if not page_date: |
| 322 | + print(">> WARN: 날짜 없음, 건너뜀") |
| 323 | + continue |
| 324 | + title, filepath = save_contribution(page, page_date, existing_map) |
| 325 | + existing_map[page["id"]] = filepath |
| 326 | + synced_files.add(filepath) |
| 327 | + print(f">> 저장: {filepath} ({title})") |
| 328 | + saved += 1 |
| 329 | + |
| 330 | + has_more = data.get("has_more", False) |
| 331 | + next_cursor = data.get("next_cursor") |
| 332 | + |
| 333 | + if fetch_mode != "DAILY": |
| 334 | + remove_orphans(synced_files) |
| 335 | + existing_map = {k: v for k, v in existing_map.items() if v in synced_files} |
| 336 | + |
| 337 | + save_sync_map(existing_map) |
| 338 | + |
| 339 | + # overview.md 카운트 갱신 (전체 추적 파일 수 기준) |
| 340 | + total = len(existing_map) |
| 341 | + update_overview_count(total) |
| 342 | + |
| 343 | + print(f">> 완료: {saved}개 저장, 누적 {total}건") |
| 344 | + |
| 345 | + |
| 346 | +if __name__ == "__main__": |
| 347 | + main() |
0 commit comments