91 lines
2.9 KiB
Python
91 lines
2.9 KiB
Python
|
import sys
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
import shutil
|
||
|
|
|
||
|
|
# Add current dir to path to import scraping scripts
|
||
|
|
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
try:
|
||
|
|
import scrape_newbt
|
||
|
|
import scrape_cbtbank
|
||
|
|
except ImportError as e:
|
||
|
|
print(f"Failed to import scraping modules: {e}")
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
# Configuration
|
||
|
|
db_file = r"./qple_quiz.db"
|
||
|
|
assets_dir = r"./frontend/public/assets"
|
||
|
|
delay = 0.1 # Moderate delay to prevent server blocking but process fast
|
||
|
|
|
||
|
|
# 1. Newbt Exams List
|
||
|
|
newbt_exams = [
|
||
|
|
"ISMS-P 인증심사원",
|
||
|
|
"AWS Solutions Architect Associate",
|
||
|
|
"Artificial Intelligence",
|
||
|
|
"DevOps and Agile",
|
||
|
|
"정보처리기사(구)",
|
||
|
|
"정보처리기사",
|
||
|
|
"PSAT 헌법",
|
||
|
|
"PSAT 자료해석",
|
||
|
|
"PSAT 상황판단",
|
||
|
|
"PSAT 언어논리",
|
||
|
|
"데이터분석 준전문가",
|
||
|
|
"리눅스마스터 1급",
|
||
|
|
"리눅스마스터 1급 실기",
|
||
|
|
"리눅스마스터 2급",
|
||
|
|
"정보시스템감리사"
|
||
|
|
]
|
||
|
|
|
||
|
|
# 2. CBTBank Exams List
|
||
|
|
cbtbank_exams = [
|
||
|
|
"정보통신기사(구)",
|
||
|
|
"컴퓨터시스템기사(A형)",
|
||
|
|
"컴퓨터시스템기사(B형)"
|
||
|
|
]
|
||
|
|
|
||
|
|
print("======================================================================")
|
||
|
|
print(">>> STARTING UNIFIED BATCH SCRAPING PIPELINE")
|
||
|
|
print("======================================================================")
|
||
|
|
|
||
|
|
# Step A: Scrape from Newbt
|
||
|
|
print(f"\n>>> [PART 1/2] Scraping {len(newbt_exams)} categories from newbt.kr...")
|
||
|
|
for idx, exam in enumerate(newbt_exams, 1):
|
||
|
|
print(f"\n----------------------------------------------------------------------")
|
||
|
|
print(f"[Newbt {idx}/{len(newbt_exams)}] SCRAPING: {exam}")
|
||
|
|
print(f"----------------------------------------------------------------------")
|
||
|
|
try:
|
||
|
|
scrape_newbt.scrape_exam(
|
||
|
|
exam_name=exam,
|
||
|
|
selected_rounds=None,
|
||
|
|
overwrite=False,
|
||
|
|
delay=delay,
|
||
|
|
db_file=db_file,
|
||
|
|
assets_dir=assets_dir
|
||
|
|
)
|
||
|
|
except Exception as e:
|
||
|
|
print(f"[Error] Failed scraping {exam} from newbt: {e}")
|
||
|
|
time.sleep(1.0)
|
||
|
|
|
||
|
|
# Step B: Scrape from CBTBank
|
||
|
|
print(f"\n>>> [PART 2/2] Scraping {len(cbtbank_exams)} categories from cbtbank.kr...")
|
||
|
|
for idx, exam in enumerate(cbtbank_exams, 1):
|
||
|
|
print(f"\n----------------------------------------------------------------------")
|
||
|
|
print(f"[CBTBank {idx}/{len(cbtbank_exams)}] SCRAPING: {exam}")
|
||
|
|
print(f"----------------------------------------------------------------------")
|
||
|
|
try:
|
||
|
|
scrape_cbtbank.scrape_exam(
|
||
|
|
exam_name=exam,
|
||
|
|
selected_rounds=None,
|
||
|
|
overwrite=False,
|
||
|
|
delay=delay,
|
||
|
|
db_file=db_file,
|
||
|
|
assets_dir=assets_dir
|
||
|
|
)
|
||
|
|
except Exception as e:
|
||
|
|
print(f"[Error] Failed scraping {exam} from cbtbank: {e}")
|
||
|
|
time.sleep(1.0)
|
||
|
|
|
||
|
|
print("\n======================================================================")
|
||
|
|
print(">>> ALL SCRAPING TASKS COMPLETE!")
|
||
|
|
print("======================================================================")
|