

형태소 분석을 위한 단어(명사) 추출 대신에 정규표현식을 통해 전처리 과정을 대신했다.
def crawling(**context):
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"}
inputURL = "https://news.naver.com/main/ranking/popularMemo.naver" #댓글 많은 뉴스 순으로, 목적에 취합, 핫토픽이기 때문에
response = requests.get(inputURL, headers=headers)
beautifulSoup = BeautifulSoup(response.content, "html.parser")
titles = beautifulSoup.find_all("a", attrs={"class":"list_title nclicks('RBP.cmtnws')"})
get_text_titles =""
for title in titles:
get_text_titles += (title.get_text())
context['task_instance'].xcom_push(key='pushed_crawling_value', value=get_text_titles)
def Pre_processing(**context):
crawled_date = context['task_instance'].xcom_pull(key='pushed_crawling_value', task_ids='crawling_data')
processed_data = re.sub('[-=+,#/\?:^$.@*\"※~&%ㆍ!』\\‘|\(\)\[\]\<\>`\'\“\”\‘\’\·…》]',' ', crawled_date).split()
word_counts = {}
for word in processed_data:
if len(word) <= 1:
continue
if word not in word_counts:
word_counts[word] = 0
word_counts[word] += 1
word_counts2 = sorted(word_counts,key=lambda x:word_counts[x], reverse=True)[:10]
context['task_instance'].xcom_push(key='processed_count_value', value=word_counts2)