18_konlpy.Okt를 활용한 감성분석(네이버 영화 리뷰)

konlpy.Okt를 활용한 감성분석
(네이버 영화 리뷰)

 

from konlpy.tag import Okt, Kkma # 성능이 좋은 한국어 형태소분석기(별도의 설치 과정이 필요함)
import pandas as pd
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow import keras
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
import re

 

 

형태소 분석기

okt = Okt()

 

# 각 형태소의 품사를 출력
print(okt.pos("아버지가 방에 들어가신다"))

[('아버지', 'Noun'), ('가', 'Josa'), ('방', 'Noun'), ('에', 'Josa'), ('들어가신다', 'Verb')]

 

 

# 형태소별로 쪼개줌
print(okt.morphs("아버지가 방에 들어가신다"))

['아버지', '가', '방', '에', '들어가신다']

 

 

print(okt.morphs("아버지가 방에 들어가신다", stem = True))

['아버지', '가', '방', '에', '들어가다']

 

 

print(okt.nouns("아버지가 방에 들어가신다"))

['아버지', '방']

 

 

kkma = Kkma()

 

print(kkma.pos("아버지가방에들어가신다"))

[('아버지', 'NNG'), ('가방', 'NNG'), ('에', 'JKM'), ('들어가', 'VV'), ('시', 'EPH'), ('ㄴ다', 'EFN')]

 

 

 

데이터 확인

train_df = pd.read_csv("./data/nsmc/ratings_train.txt", sep = "\t")
test_df = pd.read_csv("./data/nsmc/ratings_test.txt", sep = "\t")

 

train_df.head()

 

 

test_df.head()

 

 

train_df.shape, test_df.shape

((150000, 3), (50000, 3))

 

 

train_df.isna().sum()

id          0
document    5
label       0
dtype: int64

 

 

test_df.isna().sum()

id          0
document    3
label       0
dtype: int64

 

 

train_df = train_df.dropna(subset = ["document"])
test_df = test_df.dropna(subset = ["document"])

 

train_df.shape, test_df.shape

((149995, 3), (49997, 3))

 

 

# 종속변수 확인
np.unique(train_df["label"], return_counts = True)

(array([0, 1], dtype=int64), array([75170, 74825], dtype=int64))

 

 

 

데이터 전처리

# 한글 이외의 문자들 제거
train_df["document"] = train_df["document"].str.replace("[^ㄱ-ㅎㅏ-ㅣ가-힣 ]", "", regex = True)
test_df["document"] = test_df["document"].str.replace("[^ㄱ-ㅎㅏ-ㅣ가-힣 ]", "", regex = True)
# regex = True 정규표현식을 쓰겠다는 의미

 

train_df.head()

 

 

test_df.head()

 

 

# 한글 이외의 값이 있으므로 길이가 0인 컬럼이 생김
len(test_df.iloc[1, 1])

0

 

 

# document 컬럼이 비어있는 경우 삭제
train_df = train_df[train_df["document"].map(lambda x: len(x.strip()) >= 1)]
test_df = test_df[test_df["document"].map(lambda x: len(x.strip()) >= 1)]

 

train_df.shape, test_df.shape

((148740, 3), (49575, 3))

 

 

# 중복 데이터 확인
train_df[train_df["document"].duplicated(keep = False)].sort_values("document")

 

 

# 중복 제거
train_df = train_df.drop_duplicates(subset = ["document"])
test_df = test_df.drop_duplicates(subset = ["document"])

 

 

train_df.shape, test_df.shape

((143660, 3), (48403, 3))

 

 

 

토큰화

%%time
# 실행 시간 체크
train_df["token"] = train_df["document"].map(lambda x: okt.morphs(x, stem = True))
test_df["token"] = test_df["document"].map(lambda x: okt.morphs(x, stem = True))

CPU times: total: 16min 22s
Wall time: 7min 31s

 

 

train_df.to_csv("./data/nsmc_ratings_train_pre.csv", index = False)
test_df.to_csv("./data/nsmc_ratings_test_pre.csv", index = False)

train_df = pd.read_csv("./data/nsmc_ratings_train_pre.csv")
test_df = pd.read_csv("./data/nsmc_ratings_test_pre.csv")

train_df["token"] = train_df["token"].map(lambda x: eval(x))
test_df["token"] = test_df["token"].map(lambda x: eval(x))

 

 

train_df.head()

 

 

train_df = train_df[train_df["token"].map(lambda x: len(x) > 0)]
test_df = test_df[test_df["token"].map(lambda x: len(x) > 0)]

 

 

train_df.shape, test_df.shape

((143660, 4), (48403, 4))

 

 

 

정수인코딩

tokenizer = Tokenizer()
# num_words = 1000 가장 빈번하게 사용되는 상위 1000개를 인코딩

 

# 단어 집합 생성
# 등장 빈도 수가 높은 순서대로 정수값 부여
tokenizer.fit_on_texts(train_df["token"])

 

# 단어 집합
tokenizer.word_index

{'이': 1,
 '영화': 2,
 '보다': 3,
 '하다': 4,
 '의': 5,
 '에': 6,
 '가': 7,
 '을': 8,
 '도': 9,
 '들': 10,
 '는': 11,
 '를': 12,
 '은': 13,
 '없다': 14,
 '이다': 15,
 '있다': 16,
 '좋다': 17,
 '너무': 18,
 ...

 

 

# 단어 등장 수
tokenizer.word_counts

 

OrderedDict([('아', 4121),
             ('더빙', 572),
             ('진짜', 8288),
             ('짜증나다', 1002),
             ('목소리', 374),
             ('흠', 246),
             ('포스터', 572),
             ('보고', 4653),
             ('초딩', 422),
             ('영화', 50172),
             ('줄', 1240),
             ('오버', 142),
             ('연기', 6326),
             ('조차', 242),
             ('가볍다', 360),
             ('않다', 7718),
             ('너', 670),
             ('무재', 69),
             ('밓었', 1),
             ...

 

 

 

단어 등장 수 분석

total_cnt = len(tokenizer.word_index)

 

total_cnt

43770

 

 

# 토큰화
x_train = tokenizer.texts_to_sequences(train_df["token"])
x_test = tokenizer.texts_to_sequences(test_df["token"])

 

 

len(x_train)

143660

 

 

x_train[0]

[64, 473, 27, 278, 677]

 

 

# 등장 빈도가 2번 미만인 단어 수
rare_cnt = 0

for key, value in tokenizer.word_counts.items():
    if value < 2:
        rare_cnt += 1

 

 

rare_cnt

18738

 

 

rare_cnt / total_cnt

0.42810143934201506

• 1번만 등장하는 단어는 전체 42.81%

 

total_cnt - rare_cnt

25032

 

 

# 재토큰화
tokenizer = Tokenizer(num_words = 20000)
# oov_token="<OOV>" 옵션을 설정하여 희귀 단어, 신조어와 같은 문제를 완화가능

tokenizer.fit_on_texts(train_df["token"])

x_train = tokenizer.texts_to_sequences(train_df["token"])
x_test = tokenizer.texts_to_sequences(test_df["token"])

 

x_train[:5]

[[64, 473, 27, 278, 677],
 [965, 474, 54, 620, 2, 230, 1466, 37, 979, 694, 31],
 [403, 2460, 2330, 5684, 3, 237, 21, 19],
 [6507, 120, 8133, 234, 70, 11, 14, 39, 3619],
 [1041,
  19431,
  42,
  5,
  9158,
  37,
  7,
  ...

 

 

 

길이 분석

lengths = np.array([len(i) for i in train_df["token"]])

plt.figure()

plt.hist(lengths)

plt.xlabel("lengths")
plt.ylabel("frequency")
plt.show()

 

 

np.mean(lengths), np.min(lengths), np.max(lengths)

(13.234950577753027, 1, 78)

 

 

np.quantile(lengths, [0.25, 0.5, 0.75, 0.9])

array([ 6., 10., 16., 28.])

 

 

# 전체적으로 감상평이 짧으므로 모든 거의 문장을 살리는 쪽으로 진행하기 위해 20으로 설정
train_seq = pad_sequences(x_train, maxlen = 20)
test_seq = pad_sequences(x_test, maxlen = 20)

# 데이터 분할
sub_seq, val_seq, y_sub, y_val = train_test_split(train_seq, train_df["label"], test_size = 0.2,
                                                  stratify = train_df["label"], random_state = 26)

 

 

sub_seq.shape, val_seq.shape, y_sub.shape, y_val.shape

((114928, 20), (28732, 20), (114928,), (28732,))

 

test_seq.shape

(48403, 20)

 

 

모델 설계

 

model = keras.Sequential()
model.add(keras.Input(shape = (20,)))

model.add(keras.layers.Embedding(20000, 128))

model.add(keras.layers.GRU(64, dropout = 0.3, return_sequences = True))
# GRU는 입력값에 dropout을 적용하는것이기 때문에 dropout을 중복으로 지정하면 안되므로 삭제
# model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.GRU(128, dropout = 0.3))
model.add(keras.layers.Dropout(0.3))

model.add(keras.layers.Dense(128))
model.add(keras.layers.Dropout(0.3))

model.add(keras.layers.Dense(1, activation = "sigmoid"))

 

 

model.summary()

 

 

es = keras.callbacks.EarlyStopping(patience = 4, restore_best_weights = True)

 

opt = keras.optimizers.Adam(learning_rate = 1e-4)

 

model.compile(optimizer = opt, loss = "binary_crossentropy", metrics = ["accuracy"])

 

history = model.fit(sub_seq, y_sub, epochs = 100, validation_data = (val_seq, y_val),
                    batch_size = 128, callbacks = [es])