Repository navigation
Expand file tree
/
Copy pathmain.py
More file actions
121 lines (95 loc) · 4.1 KB
/
Copy pathmain.py
File metadata and controls
121 lines (95 loc) · 4.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
from fastapi import FastAPI
from transformers import pipeline
import uvicorn
from model import UrlItem, TextItem, CommentList
app = FastAPI()
def textProcessing( text: str) -> str:
text = text.replace(u'\n', u' ')
text = text.replace(u'\'', u' ')
text = text.replace(u'\\\'', u' ')
text = text.replace(u'[', u'')
text = text.replace(u']', u'')
text = text.replace(u'\xa0', u' ')
text = text.replace(u'\"', u'')
text = text.replace(u'\'', u'')
text = text.replace(u' ', u' ')
text = text.replace(u' ', u' ')
text = text.replace(u' ', u' ')
return text.strip()
# fast api 시작후 모델 로드및 초기화
class SummarizeModel:
summarizer = None
def __init__(self):
self.summarizer = pipeline("summarization", model="galaxyhm/kobartv2-summarizer-using_data",
tokenizer='galaxyhm/kobartv2-summarizer-using_data')
class BinaryTextClassificationModel:
classifier = None
def __init__(self):
self.classifier = pipeline( model='daekeun-ml/koelectra-small-v3-nsmc', tokenizer='daekeun-ml/koelectra-small-v3-nsmc')
binary_model = BinaryTextClassificationModel()
summarize_model = SummarizeModel()
# 크롤링후 테스트요약
# @app.post("/crawl/naver/")
# async def crawl(url: UrlItem):
# list_list = NewsCrawler.navercrawl(url.url)
# if len(list_list) == 7:
# a, b, c, d, e, f, g = list_list
# else:
# a,b,c,d,e,f = list_list
# text = model.summarizer(d, min_length=32, max_length=len(d) / 2)
# # print(type(text))
# # print(text[0])
# text[0]['summary_text'] = NewsCrawler.textProcessing(text[0]['summary_text'])
# text.append({'text': d})
# return {'message': text}
# 입력 글자수 체크 밑 로직 미완성
def check_and_summarize(input_text, model_pipeline, max_length):
text_length = len(input_text)
text_start_pointer = 0
text_end_pointer = 0
return_text = []
while True:
if max_length < text_length - text_start_pointer:
if max_length > text_length - text_end_pointer:
text_end_pointer = text_length
else:
text_end_pointer = text_start_pointer + max_length
min_text = model_pipeline(input_text[text_start_pointer:text_end_pointer])
return_text.append(min_text)
print(min_text)
# return_text.append(model_pipeline.summarizer(input_text[text_start_pointer:text_start_pointer+1999]))
text_start_pointer = text_start_pointer+max_length-5
else:
# return_text.append(model_pipeline(input_text[text_start_pointer:]))
min_text = model_pipeline(input_text[text_start_pointer:])
return_text.append(min_text)
print(min_text)
# return_text.append(model_pipeline.summarizer(input_text[text_start_pointer:]))
return return_text
@app.post("/summarize/text")
async def summarize_text(text: TextItem):
after_preproces = textProcessing(text.text)
textmodel = summarize_model.summarizer(after_preproces, min_length=32, max_length=len(after_preproces) / 2)
return {'message': textmodel}
@app.post('/emotion/text')
async def emotion_text(received_comment: CommentList):
only_comment_list = []
received_json = received_comment.dict()
positive_num = 0
for i in received_json['comments']:
only_comment_list.append(i['contents'])
classifier_values = binary_model.classifier(only_comment_list)
for i in range(len(received_json['comments'])):
received_json['comments'][i]['emotion'] = classifier_values[i]['label']
received_json['comments'][i]['emotion_value'] = classifier_values[i]['score']
if classifier_values[i]['label'] == '1':
positive_num += 1
positive_ratio = positive_num / len(received_json['comments'])
negative_ratio = 1 - positive_ratio
received_json['whole_emotion_positive_ratio'] = positive_ratio
received_json['whole_emotion_negative_ratio'] = negative_ratio
return received_json
def main():
uvicorn.run(app, host='0.0.0.0', port=8908)
if __name__ == '__main__':
main()