Selenium <-> Requests간 쿠키 사용하기

웹 크롤링 시 Selenium과 requests에서 상호 쿠키를 사용할 필요가 있다.
아래 코드로 셀레니움으로 로그인 및 쿠키를 저장하고 활용해보자.

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager

import requests
import time
import pickle

def open_Driver():
    options = Options()

    options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
    options.add_argument('lang=ko_KR')
    options.add_argument('--window-size=430,932')
    options.add_argument('--disable-gpu')
    options.add_argument('--disable-infobars')
    options.add_argument('--disable-extensions')
    options.add_argument('--disable-blink-features=AutomationControlled')
    options.add_argument('--disable-automation')

    service = Service(executable_path=ChromeDriverManager().install())
    driver = webdriver.Chrome(service=service, options=options)

    return driver

# 쿠키 저장 함수
def save_cookie(driver):
    url = 'https://m.naver.com/'
    driver.get(url)

    # 30초 이내 로그인
    time.sleep(30)
    pickle.dump(driver.get_cookies(), open('naver_cookies.pkl', 'wb')) 

    driver.quit()

# 셀레니움에서 쿠키 정상 동작 확인
def check_login_selenium(driver):
    url = 'https://m.naver.com'
    driver.get(url)

    cookies = pickle.load(open('naver_cookies.pkl', 'rb'))
    for cookie in cookies:
        driver.add_cookie(cookie)

    driver.get(url)
    time.sleep(30)
    driver.quit()

# request에서 쿠키 정상 동작 확인
def check_login_request():
    url = 'https://m.naver.com'

    header = {
        'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
    }

    cookies = pickle.load(open('naver_cookies.pkl', 'rb'))

    s = requests.Session()
    s.headers.update(header)

    for cookie in cookies:
        c = {cookie['name'] : cookie['value']}
        s.cookies.update(c)

    response = s.request('GET', url)

    if response.text.find('MM_PROFILE_IMAGE') != -1:
        print('Login success!')

def main():
    # 1. 최초 한번만 실행, 이후 주석 처리
    driver = open_Driver()
    save_cookie(driver)

    # 2. 쿠키로 셀레니움 로그인 여부 확인
    #driver = open_Driver()
    #check_login_selenium(driver)

    # 3. 쿠키로 request로 로그인 여부 확인
    #check_login_request()

if __name__ == '__main__':
    main()

참고 링크
https://sir.kr/pg_tip/17077
https://m.blog.naver.com/draco6/221664143794