이번에 소개할 프로젝트는
deep learing을 이용해서 채팅 로봇이 사람의 감정을 판단 해주는 프로젝트를 만들어 보앗습니다
코드로 넘어가면.......
첫번째로! 우리가 필요한 모듈을 가져왓습니다
저희가 딥러닝 모델을 사용하기 때문에 트레이닝된 딥러닝 모델의 틀을 가져옵니다
class CNNkkk(nn.Module): def __init__(self): super(CNNkkk,self).__init__() #48*48 self.layer1 = nn.Sequential( nn.Conv2d(1,64,kernel_size=3,stride=1,padding=1), nn.Conv2d(64,64,kernel_size=3,stride=1,padding=1), nn.BatchNorm2d(64), nn.LeakyReLU(), nn.MaxPool2d(2,2) ) #24*24 self.layer2 = nn.Sequential( nn.Conv2d(64,64*2,kernel_size=3,stride=1,padding=1), nn.Conv2d(64*2,64*2,kernel_size=3,stride=1,padding=1), nn.BatchNorm2d(64*2),nn.LeakyReLU(),
nn.MaxPool2d(2,2) ) #12*12 self.layer3 = nn.Sequential( nn.Conv2d(64*2,64*4,kernel_size=3,stride=1,padding=1), nn.Conv2d(64*4,64*4,kernel_size=3,stride=1,padding=1), nn.BatchNorm2d(64*4), nn.LeakyReLU(), nn.MaxPool2d(2,2) ) self.layer4 = nn.Sequential( nn.Conv2d(64*4,64*6,kernel_size=3,stride=1,padding=1), nn.Conv2d(64*6,64*6,kernel_size=3,stride=1,padding=1), nn.Conv2d(64*6,64*6,kernel_size=3,stride=1,padding=1), nn.BatchNorm2d(64*6), nn.LeakyReLU(), nn.MaxPool2d(2,2), ) #6*6 self.fc1 = nn.Sequential( nn.Linear(6*3*64*3,1024), nn.ReLU(), ) self.fc2 = nn.Sequential( nn.Linear(1024,512), nn.ReLU(), ) self.fc3 = nn.Sequential( nn.Linear(512,7), ) def forward(self,x): x1= self.layer1(x) x2 = self.layer2(x1) x3 = self.layer3(x2) x3 = self.layer4(x3) x5 = x3.view(-1,6*3*64*3) output = self.fc1(x5) output = self.fc2(output) output = self.fc3(output) return output학습되 모델의 weight를 가져옵니다
model = torch.load('emotion_model46_166.pkl')chat Bot을 사용하기 위한 함수와 코드를 작성합니다
from telegram.ext import Updater, MessageHandler, Filters ,CommandHandler,CallbackQueryHandlerfrom telegram import InlineKeyboardButton, InlineKeyboardMarkupimport cv2,timeimport pandas as pdimport numpy as npdir_now = os.path.dirname(os.path.abspath('chatbot'))
face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')emotion_box = {0:'화난', 1:'싫은', 2:'공포', 3:'행복한', 4:'슬픈', 5:'놀라운', 6:'자연스러운'}emotion_boxs = {'화난':0,'싫은':1,'공포':2,'행복한':3,'슬픈':4,'놀라운':5, '자연스러운':6,'잘모르겟어':7}def predict_face(image): read_image = cv2.imread(image) read_image = cv2.cvtColor(read_image, cv2.COLOR_BGR2GRAY) read_image = read_image.reshape(1,1, read_image.shape[0], read_image.shape[1])read_image_final = read_image/255.0
gray_Z = torch.from_numpy(read_image_final) gray_Y = torch.from_numpy(np.ones(1)) gray_test = TensorDataset(gray_Z,gray_Y) grayloader = DataLoader(dataset=gray_test, batch_size=1) for data in grayloader: image,y = data img111 = Variable(image.float()) outputs = model(img111) _, predicted = torch.max(outputs, 1) return int(predicted)def build_button(text_list, callback_header = ""): button_list = [] text_header = callback_header if callback_header != "" : text_header += "," for text in text_list: button_list.append(InlineKeyboardButton(text, callback_data=text_header + text)) return button_listdef build_menu(buttons, n_cols, header_buttons=None, footer_buttons=None): menu = [buttons[i:i + n_cols] for i in range(0, len(buttons), n_cols)] if header_buttons: menu.insert(0, header_buttons) if footer_buttons: menu.append(footer_buttons) return menudef introuduce(bot, update) : content = '안녕! 나는 감정을 분석해주는 친구야!'+"\n"+'나를 사용할려면 /start를 적어줘' update.message.reply_text(content)def start(bot,update): update.message.reply_text("감정을 분석하고 싶은 사진을 나한테 보내줄래?")def get_photo(bot, update): global number number = update.message.message_id file_path = os.path.join(dir_now, 'image/from_telegram_%s.jpg'%(number)) photo_id = update.message.photo[-1].file_id photo_file = bot.getFile(photo_id) photo_file.download(file_path) update.message.reply_text('잠시만 기다려줘 감정 분석 중이야!') image = cv2.imread('image/from_telegram_%s.jpg'%(number)) gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY) face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml') faces = face_cascade.detectMultiScale(gray, 1.3, 5) if len(faces) == 0: update.message.reply_text('다른 사진을 보내주면 안될까?') return else: update.message.reply_text('ing...') for (x,y,w,h) in faces: face_clip = image[y:y+h, x:x+w] cv2.imwrite('image_data/from_telegram_%s.jpg'%(number), cv2.resize(face_clip, (48, 48))) time.sleep(3) answer = predict_face('image_data/from_telegram_%s.jpg'%(number)) update.message.reply_text('내가 봣을때는 '+ emotion_box[answer] + ' 표정인것 같아!') show_list = [] show_list.append(InlineKeyboardButton("맞아", callback_data="맞아")) # add on button show_list.append(InlineKeyboardButton("아니야", callback_data="아니야")) # add off button show_list.append(InlineKeyboardButton("모르겟어", callback_data="모르겟어")) # add cancel button show_markup = InlineKeyboardMarkup(build_menu(show_list, len(show_list))) # make markup update.message.reply_text("아니야?", reply_markup=show_markup) def callback_get(bot, update): data_selected = update.callback_query.data if data_selected.find("모르겟어") != -1 : bot.edit_message_text(text="너도 잘 모르는구나 ㅐ.ㅐ", chat_id=update.callback_query.message.chat_id, message_id=update.callback_query.message.message_id) return elif data_selected.find("맞아") != -1 : bot.edit_message_text(text="고마워^^", chat_id=update.callback_query.message.chat_id, message_id=update.callback_query.message.message_id) return if len(data_selected.split(",")) == 1 : button_list = build_button(["화난", "싫은", "공포",'행복한','슬픈','놀라운','자연스러운', "잘모르겟어"], data_selected) show_markup = InlineKeyboardMarkup(build_menu(button_list, len(button_list)-4)) bot.edit_message_text(text="좀더 노력할게"+'\n'+'혹시 무슨 표정인거야?', chat_id=update.callback_query.message.chat_id, message_id=update.callback_query.message.message_id, reply_markup=show_markup) elif len(data_selected.split(",")) == 2 : box = update.callback_query.data.split(',') test = pd.read_csv('chatbot_data.csv') img = cv2.imread('image_data/from_telegram_%s.jpg'%(number)) img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) img=img.reshape(1,48*48) img_data = pd.DataFrame(img) label = np.array([emotion_boxs[box[-1]]]) label = pd.DataFrame({'label':label}) image = pd.concat([img_data,label],axis=1) image.to_csv('chatbot_data1.csv',index=False) train = pd.read_csv('chatbot_data1.csv') test= pd.concat([test,train],axis=0) test.to_csv('chatbot_data.csv',index=False) bot.edit_message_text(text="{} 표정 인거구나 고마워 :)".format(box[-1]), chat_id=update.callback_query.message.chat_id,message_id=update.callback_query.message.message_id)
채팅로봇 실행하기!
def main(): updater = Updater(my_token) updater.dispatcher.add_handler(MessageHandler(Filters.text, introuduce)) updater.dispatcher.add_handler(CommandHandler('start', start)) updater.dispatcher.add_handler(MessageHandler(Filters.photo, get_photo)) updater.dispatcher.add_handler(CallbackQueryHandler(callback_get)) updater.start_polling(timeout=3, clean=True) updater.idle() main()채팅 로봇이 봣을때와 사용자가 다를때 사용자가 봣을때 그 감정은 이 감정이야 라고 알려주면 나중에 업데이트 될수 잇게 csv 파일로 저장한다
test = pd.read_csv('chatbot_data.csv')img = cv2.imread('image_data/from_telegram.jpg')img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)img=img.reshape(1,48*48)img_data = pd.DataFrame(img)label = np.array([emotion_boxs[box[-1]]])label = pd.DataFrame({'label':label})image = pd.concat([img_data,label],axis=1)image.to_csv('chatbot_data1.csv',index=False)train = pd.read_csv('chatbot_data1.csv')test= pd.concat([test,train],axis=0)test.to_csv('chatbot_data.csv',index=False)