py

2026-06-27 02:21:09 +00:00
parent f5ddd0b877
commit 4c2ce929d9
10 changed files with 1462 additions and 1 deletions
@@ -0,0 +1,90 @@
+import sys
+import os
+import time
+import shutil
+
+# Add current dir to path to import scraping scripts
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+try:
+    import scrape_newbt
+    import scrape_cbtbank
+except ImportError as e:
+    print(f"Failed to import scraping modules: {e}")
+    sys.exit(1)
+
+# Configuration
+db_file = r"./qple_quiz.db"
+assets_dir = r"./frontend/public/assets"
+delay = 0.1  # Moderate delay to prevent server blocking but process fast
+
+# 1. Newbt Exams List
+newbt_exams = [
+    "ISMS-P 인증심사원",
+    "AWS Solutions Architect Associate",
+    "Artificial Intelligence",
+    "DevOps and Agile",
+    "정보처리기사(구)",
+    "정보처리기사",
+    "PSAT 헌법",
+    "PSAT 자료해석",
+    "PSAT 상황판단",
+    "PSAT 언어논리",
+    "데이터분석 준전문가",
+    "리눅스마스터 1급",
+    "리눅스마스터 1급 실기",
+    "리눅스마스터 2급",
+    "정보시스템감리사"
+]
+
+# 2. CBTBank Exams List
+cbtbank_exams = [
+    "정보통신기사(구)",
+    "컴퓨터시스템기사(A형)",
+    "컴퓨터시스템기사(B형)"
+]
+
+print("======================================================================")
+print(">>> STARTING UNIFIED BATCH SCRAPING PIPELINE")
+print("======================================================================")
+
+# Step A: Scrape from Newbt
+print(f"\n>>> [PART 1/2] Scraping {len(newbt_exams)} categories from newbt.kr...")
+for idx, exam in enumerate(newbt_exams, 1):
+    print(f"\n----------------------------------------------------------------------")
+    print(f"[Newbt {idx}/{len(newbt_exams)}] SCRAPING: {exam}")
+    print(f"----------------------------------------------------------------------")
+    try:
+        scrape_newbt.scrape_exam(
+            exam_name=exam,
+            selected_rounds=None,
+            overwrite=False,
+            delay=delay,
+            db_file=db_file,
+            assets_dir=assets_dir
+        )
+    except Exception as e:
+        print(f"[Error] Failed scraping {exam} from newbt: {e}")
+    time.sleep(1.0)
+
+# Step B: Scrape from CBTBank
+print(f"\n>>> [PART 2/2] Scraping {len(cbtbank_exams)} categories from cbtbank.kr...")
+for idx, exam in enumerate(cbtbank_exams, 1):
+    print(f"\n----------------------------------------------------------------------")
+    print(f"[CBTBank {idx}/{len(cbtbank_exams)}] SCRAPING: {exam}")
+    print(f"----------------------------------------------------------------------")
+    try:
+        scrape_cbtbank.scrape_exam(
+            exam_name=exam,
+            selected_rounds=None,
+            overwrite=False,
+            delay=delay,
+            db_file=db_file,
+            assets_dir=assets_dir
+        )
+    except Exception as e:
+        print(f"[Error] Failed scraping {exam} from cbtbank: {e}")
+    time.sleep(1.0)
+
+print("\n======================================================================")
+print(">>> ALL SCRAPING TASKS COMPLETE!")
+print("======================================================================")