sushant07 commited on
Commit
1b93b91
1 Parent(s): d53319c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +505 -0
app.py CHANGED
@@ -0,0 +1,505 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ # coding: utf-8
3
+
4
+ # In[1]:
5
+ import validators, re
6
+ import torch
7
+ from fake_useragent import UserAgent
8
+ from bs4 import BeautifulSoup
9
+ import streamlit as st
10
+ from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer
11
+ from sentence_transformers import SentenceTransformer
12
+ import en_core_web_lg
13
+ import time
14
+ import base64
15
+ import requests
16
+ import docx2txt
17
+ from io import StringIO
18
+ from PyPDF2 import PdfFileReader
19
+ import warnings
20
+ import nltk
21
+ import itertools
22
+ import numpy as np
23
+
24
+ nltk.download('punkt')
25
+
26
+ from nltk import sent_tokenize
27
+
28
+ warnings.filterwarnings("ignore")
29
+
30
+
31
+ # In[2]:
32
+
33
+ time_str = time.strftime("%d%m%Y-%H%M%S")
34
+ HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
35
+ margin-bottom: 2.5rem">{}</div> """
36
+ #Functions
37
+
38
+ def article_text_extractor(url: str):
39
+
40
+ '''Extract text from url and divide text into chunks if length of text is more than 500 words'''
41
+
42
+ ua = UserAgent()
43
+
44
+ headers = {'User-Agent':str(ua.chrome)}
45
+
46
+ r = requests.get(url,headers=headers)
47
+
48
+ soup = BeautifulSoup(r.text, "html.parser")
49
+ title_text = soup.find_all(["h1"])
50
+ para_text = soup.find_all(["p"])
51
+ article_text = [result.text for result in para_text]
52
+
53
+ try:
54
+
55
+ article_header = [result.text for result in title_text][0]
56
+
57
+ except:
58
+
59
+ article_header = ''
60
+
61
+ article = nlp(" ".join(article_text))
62
+ sentences = [i.text for i in list(article.sents)]
63
+
64
+ current_chunk = 0
65
+ chunks = []
66
+
67
+ for sentence in sentences:
68
+ if len(chunks) == current_chunk + 1:
69
+ if len(chunks[current_chunk]) + len(sentence.split(" ")) <= 500:
70
+ chunks[current_chunk].extend(sentence.split(" "))
71
+ else:
72
+ current_chunk += 1
73
+ chunks.append(sentence.split(" "))
74
+ else:
75
+ chunks.append(sentence.split(" "))
76
+
77
+ for chunk_id in range(len(chunks)):
78
+ chunks[chunk_id] = " ".join(chunks[chunk_id])
79
+
80
+ return article_header, chunks
81
+
82
+ def chunk_clean_text(text):
83
+
84
+ """Chunk text longer than 500 tokens"""
85
+
86
+ article = nlp(text)
87
+ sentences = [i.text for i in list(article.sents)]
88
+
89
+ current_chunk = 0
90
+ chunks = []
91
+
92
+ for sentence in sentences:
93
+ if len(chunks) == current_chunk + 1:
94
+ if len(chunks[current_chunk]) + len(sentence.split(" ")) <= 500:
95
+ chunks[current_chunk].extend(sentence.split(" "))
96
+ else:
97
+ current_chunk += 1
98
+ chunks.append(sentence.split(" "))
99
+ else:
100
+ chunks.append(sentence.split(" "))
101
+
102
+ for chunk_id in range(len(chunks)):
103
+ chunks[chunk_id] = " ".join(chunks[chunk_id])
104
+
105
+ return chunks
106
+
107
+ def preprocess_plain_text(x):
108
+
109
+ x = x.encode("ascii", "ignore").decode() # unicode
110
+ x = re.sub(r"https*\S+", " ", x) # url
111
+ x = re.sub(r"@\S+", " ", x) # mentions
112
+ x = re.sub(r"#\S+", " ", x) # hastags
113
+ x = re.sub(r"\s{2,}", " ", x) # over spaces
114
+ x = re.sub("[^.,!?A-Za-z0-9]+", " ", x) # special charachters except .,!?
115
+
116
+ return x
117
+
118
+ def extract_pdf(file):
119
+
120
+ '''Extract text from PDF file'''
121
+
122
+ pdfReader = PdfFileReader(file)
123
+ count = pdfReader.numPages
124
+ all_text = ""
125
+ for i in range(count):
126
+ page = pdfReader.getPage(i)
127
+ all_text += page.extractText()
128
+
129
+
130
+ return all_text
131
+
132
+
133
+ def extract_text_from_file(file):
134
+
135
+ '''Extract text from uploaded file'''
136
+
137
+ # read text file
138
+ if file.type == "text/plain":
139
+ # To convert to a string based IO:
140
+ stringio = StringIO(file.getvalue().decode("utf-8"))
141
+
142
+ # To read file as string:
143
+ file_text = stringio.read()
144
+
145
+ # read pdf file
146
+ elif file.type == "application/pdf":
147
+ file_text = extract_pdf(file)
148
+
149
+ # read docx file
150
+ elif (
151
+ file.type
152
+ == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
153
+ ):
154
+ file_text = docx2txt.process(file)
155
+
156
+ return file_text
157
+
158
+ def summary_downloader(raw_text):
159
+
160
+ b64 = base64.b64encode(raw_text.encode()).decode()
161
+ new_filename = "new_text_file_{}_.txt".format(time_str)
162
+ st.markdown("#### Download Summary as a File ###")
163
+ href = f'<a href="data:file/txt;base64,{b64}" download="{new_filename}">Click to Download!!</a>'
164
+ st.markdown(href,unsafe_allow_html=True)
165
+
166
+ def get_all_entities_per_sentence(text):
167
+ doc = nlp(''.join(text))
168
+
169
+ sentences = list(doc.sents)
170
+
171
+ entities_all_sentences = []
172
+ for sentence in sentences:
173
+ entities_this_sentence = []
174
+
175
+ # SPACY ENTITIES
176
+ for entity in sentence.ents:
177
+ entities_this_sentence.append(str(entity))
178
+
179
+ # FLAIR ENTITIES (CURRENTLY NOT USED)
180
+ # sentence_entities = Sentence(str(sentence))
181
+ # tagger.predict(sentence_entities)
182
+ # for entity in sentence_entities.get_spans('ner'):
183
+ # entities_this_sentence.append(entity.text)
184
+
185
+ # XLM ENTITIES
186
+ entities_xlm = [entity["word"] for entity in ner_model(str(sentence))]
187
+ for entity in entities_xlm:
188
+ entities_this_sentence.append(str(entity))
189
+
190
+ entities_all_sentences.append(entities_this_sentence)
191
+
192
+ return entities_all_sentences
193
+
194
+ def get_all_entities(text):
195
+ all_entities_per_sentence = get_all_entities_per_sentence(text)
196
+ return list(itertools.chain.from_iterable(all_entities_per_sentence))
197
+
198
+ def get_and_compare_entities(article_content,summary_output):
199
+
200
+ all_entities_per_sentence = get_all_entities_per_sentence(article_content)
201
+ entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
202
+
203
+ all_entities_per_sentence = get_all_entities_per_sentence(summary_output)
204
+ entities_summary = list(itertools.chain.from_iterable(all_entities_per_sentence))
205
+
206
+ matched_entities = []
207
+ unmatched_entities = []
208
+ for entity in entities_summary:
209
+ if any(entity.lower() in substring_entity.lower() for substring_entity in entities_article):
210
+ matched_entities.append(entity)
211
+ elif any(
212
+ np.inner(sentence_embedding_model.encode(entity, show_progress_bar=False),
213
+ sentence_embedding_model.encode(art_entity, show_progress_bar=False)) > 0.9 for
214
+ art_entity in entities_article):
215
+ matched_entities.append(entity)
216
+ else:
217
+ unmatched_entities.append(entity)
218
+
219
+ matched_entities = list(dict.fromkeys(matched_entities))
220
+ unmatched_entities = list(dict.fromkeys(unmatched_entities))
221
+
222
+ matched_entities_to_remove = []
223
+ unmatched_entities_to_remove = []
224
+
225
+ for entity in matched_entities:
226
+ for substring_entity in matched_entities:
227
+ if entity != substring_entity and entity.lower() in substring_entity.lower():
228
+ matched_entities_to_remove.append(entity)
229
+
230
+ for entity in unmatched_entities:
231
+ for substring_entity in unmatched_entities:
232
+ if entity != substring_entity and entity.lower() in substring_entity.lower():
233
+ unmatched_entities_to_remove.append(entity)
234
+
235
+ matched_entities_to_remove = list(dict.fromkeys(matched_entities_to_remove))
236
+ unmatched_entities_to_remove = list(dict.fromkeys(unmatched_entities_to_remove))
237
+
238
+ for entity in matched_entities_to_remove:
239
+ matched_entities.remove(entity)
240
+ for entity in unmatched_entities_to_remove:
241
+ unmatched_entities.remove(entity)
242
+
243
+ return matched_entities, unmatched_entities
244
+
245
+ def highlight_entities(article_content,summary_output):
246
+
247
+ markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
248
+ markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
249
+ markdown_end = "</mark>"
250
+
251
+ matched_entities, unmatched_entities = get_and_compare_entities(article_content,summary_output)
252
+
253
+ print(summary_output)
254
+
255
+ for entity in matched_entities:
256
+ summary_output = re.sub(f'({entity})(?![^rgb\(]*\))',markdown_start_green + entity + markdown_end,summary_output)
257
+
258
+ for entity in unmatched_entities:
259
+ summary_output = re.sub(f'({entity})(?![^rgb\(]*\))',markdown_start_red + entity + markdown_end,summary_output)
260
+
261
+ print("")
262
+ print(summary_output)
263
+
264
+ print("")
265
+ print(summary_output)
266
+
267
+ soup = BeautifulSoup(summary_output, features="html.parser")
268
+
269
+ return HTML_WRAPPER.format(soup)
270
+
271
+
272
+ def clean_text(text,doc=False,plain_text=False,url=False):
273
+ """Return clean text from the various input sources"""
274
+
275
+ if url:
276
+ is_url = validators.url(text)
277
+
278
+ if is_url:
279
+ # complete text, chunks to summarize (list of sentences for long docs)
280
+ article_title,chunks = article_text_extractor(url=url_text)
281
+
282
+ return article_title, chunks
283
+
284
+ elif doc:
285
+
286
+ clean_text = chunk_clean_text(preprocess_plain_text(extract_text_from_file(text)))
287
+
288
+ return None, clean_text
289
+
290
+ elif plain_text:
291
+
292
+ clean_text = chunk_clean_text(preprocess_plain_text(text))
293
+
294
+ return None, clean_text
295
+
296
+
297
+ @st.experimental_singleton(suppress_st_warning=True)
298
+ def get_spacy():
299
+ nlp = en_core_web_lg.load()
300
+ return nlp
301
+
302
+ @st.experimental_singleton(suppress_st_warning=True)
303
+ def facebook_model():
304
+ model_name = 'facebook/bart-large-cnn'
305
+ summarizer = pipeline('summarization',model=model_name,tokenizer=model_name,
306
+ device=0 if torch.cuda.is_available() else -1)
307
+ return summarizer
308
+
309
+ @st.experimental_singleton(suppress_st_warning=True)
310
+ def schleifer_model():
311
+ model_name = 'sshleifer/distilbart-cnn-12-6'
312
+ summarizer = pipeline('summarization',model=model_name, tokenizer=model_name,
313
+ device=0 if torch.cuda.is_available() else -1)
314
+ return summarizer
315
+
316
+ @st.experimental_singleton(suppress_st_warning=True)
317
+ def google_model():
318
+ model_name = 'google/pegasus-large'
319
+ summarizer = pipeline('summarization',model=model_name, tokenizer=model_name,
320
+ device=0 if torch.cuda.is_available() else -1)
321
+ return summarizer
322
+
323
+ @st.experimental_singleton(suppress_st_warning=True)
324
+ def get_sentence_embedding_model():
325
+ return SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
326
+
327
+ @st.experimental_singleton(suppress_st_warning=True)
328
+ def get_ner_pipeline():
329
+ tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
330
+ model = AutoModelForTokenClassification.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
331
+ return pipeline("ner", model=model, tokenizer=tokenizer, grouped_entities=True)
332
+
333
+ # Load all different models (cached) at start time of the hugginface space
334
+ sentence_embedding_model = get_sentence_embedding_model()
335
+ ner_model = get_ner_pipeline()
336
+ nlp = get_spacy()
337
+
338
+ #Streamlit App
339
+
340
+ st.title("Article Text and Link Extractive Summarizer with Entity Matching 📝")
341
+
342
+ model_type = st.sidebar.selectbox(
343
+ "Model type", options=["Facebook-Bart", "Sshleifer-DistilBart","Google-Pegasus"]
344
+ )
345
+
346
+ max_len= st.sidebar.slider("Maximum length of the summarized text",min_value=100,max_value=500,step=10)
347
+ min_len= st.sidebar.slider("Minimum length of the summarized text",min_value=50,max_value=200,step=10)
348
+
349
+ st.markdown(
350
+ "Model Source: [Facebook-Bart-large-CNN](https://huggingface.co/facebook/bart-large-cnn), [Sshleifer-distilbart-cnn-12-6](https://huggingface.co/sshleifer/distilbart-cnn-12-6) and [Google-Pegasus-large](https://huggingface.co/google/pegasus-large)"
351
+ )
352
+
353
+ st.markdown(
354
+ """The app supports extractive summarization which aims to identify the salient information that is then extracted and grouped together to form a concise summary.
355
+ For documents or text that is more than 500 words long, the app will divide the text into chunks and summarize each chunk. Please note when using the sidebar slider, those values represent the min/max text length per chunk of text to be summarized. If your article to be summarized is 1000 words, it will be divided into two chunks of 500 words first then the default max length of 100 words is applied per chunk, resulting in a summarized text with 200 words maximum.
356
+ There are two models available to choose from:""")
357
+
358
+ st.markdown("""
359
+ - Facebook-Bart, trained on large [CNN and Daily Mail](https://huggingface.co/datasets/cnn_dailymail) news articles.
360
+ - Sshleifer-Distilbart, which is a distilled (smaller) version of the large Bart model.
361
+ - Google Pegasus, trained on large C4 and HugeNews articles"""
362
+ )
363
+
364
+ st.markdown("""Please do note that the model will take longer to generate summaries for documents that are too long.""")
365
+
366
+ st.markdown(
367
+ "The app only ingests the below formats for summarization task:"
368
+ )
369
+ st.markdown(
370
+ """- Raw text entered in text box.
371
+ - URL of an article to be summarized.
372
+ - Documents with .txt, .pdf or .docx file formats."""
373
+ )
374
+
375
+ st.markdown("---")
376
+
377
+ if "text_area" not in st.session_state:
378
+ st.session_state.text_area = ''
379
+
380
+ if "summ_area" not in st.session_state:
381
+ st.session_state.summ_area = ''
382
+
383
+ url_text = st.text_input("Please Enter a url here")
384
+
385
+ st.markdown(
386
+ "<h3 style='text-align: center; color: red;'>OR</h3>",
387
+ unsafe_allow_html=True,
388
+ )
389
+
390
+ plain_text = st.text_area("Please Paste/Enter plain text here",)
391
+
392
+ st.markdown(
393
+ "<h3 style='text-align: center; color: red;'>OR</h3>",
394
+ unsafe_allow_html=True,
395
+ )
396
+
397
+ upload_doc = st.file_uploader(
398
+ "Upload a .txt, .pdf, .docx file for summarization"
399
+ )
400
+
401
+ if url_text:
402
+ article_title, cleaned_text = clean_text(url_text, url=True)
403
+ st.session_state.text_area = cleaned_text[0]
404
+
405
+ elif plain_text:
406
+ article_title, cleaned_text = clean_text(plain_text,plain_text=True)
407
+ st.session_state.text_area = ''.join(cleaned_text)
408
+
409
+ elif upload_doc:
410
+ article_title, cleaned_text = clean_text(upload_doc,doc=True)
411
+ st.session_state.text_area = ''.join(cleaned_text)
412
+
413
+ article_text = st.text_area(
414
+ label='Full Article Text',
415
+ placeholder="Full article text will be displayed here..",
416
+ height=250,
417
+ key='text_area'
418
+ )
419
+
420
+ summarize = st.button("Summarize")
421
+
422
+ # called on toggle button [summarize]
423
+ if summarize:
424
+ if model_type == "Facebook-Bart":
425
+ if url_text:
426
+ text_to_summarize =cleaned_text[0]
427
+ else:
428
+ text_to_summarize = cleaned_text
429
+
430
+ with st.spinner(
431
+ text="Loading Facebook-Bart Model and Extracting summary. This might take a few seconds depending on the length of your text..."
432
+ ):
433
+ summarizer_model = facebook_model()
434
+ summarized_text = summarizer_model(text_to_summarize, max_length=max_len, min_length=min_len,clean_up_tokenization_spaces=True,no_repeat_ngram_size=4)
435
+ summarized_text = ' '.join([summ['summary_text'] for summ in summarized_text])
436
+
437
+
438
+ elif model_type == "Sshleifer-DistilBart":
439
+ if url_text:
440
+ text_to_summarize = cleaned_text[0]
441
+ else:
442
+ text_to_summarize = cleaned_text
443
+
444
+ with st.spinner(
445
+ text="Loading Sshleifer-DistilBart Model and Extracting summary. This might take a few seconds depending on the length of your text..."
446
+ ):
447
+ summarizer_model = schleifer_model()
448
+ summarized_text = summarizer_model(text_to_summarize, max_length=max_len, min_length=min_len,clean_up_tokenization_spaces=True,no_repeat_ngram_size=4)
449
+ summarized_text = ' '.join([summ['summary_text'] for summ in summarized_text])
450
+
451
+ elif model_type == "Google-Pegasus":
452
+ if url_text:
453
+ text_to_summarize = cleaned_text[0]
454
+
455
+ else:
456
+ text_to_summarize = cleaned_text
457
+
458
+ with st.spinner(
459
+ text="Loading Google-Pegasus Model and Extracting summary. This might take a few seconds depending on the length of your text..."
460
+ ):
461
+ summarizer_model = google_model()
462
+ summarized_text = summarizer_model(text_to_summarize, max_length=max_len, min_length=min_len,clean_up_tokenization_spaces=True,no_repeat_ngram_size=4)
463
+ summarized_text = ' '.join([summ['summary_text'] for summ in summarized_text])
464
+
465
+ with st.spinner("Calculating and matching entities, this takes a few seconds..."):
466
+ entity_match_html = highlight_entities(text_to_summarize,summarized_text)
467
+ st.markdown("####")
468
+ print(entity_match_html)
469
+
470
+ if article_title:
471
+
472
+ # view summarized text (expander)
473
+ st.markdown(f"Article title: {article_title}")
474
+
475
+ st.session_state.summ_area = summarized_text
476
+
477
+ st.subheader('Summarized Text with no Entity Matching')
478
+
479
+ summarized_text = st.text_area(
480
+ label = '',
481
+ placeholder="Full summarized text will be displayed here..",
482
+ height=250,
483
+ key='summ_area'
484
+ )
485
+
486
+ st.markdown("####")
487
+
488
+ st.subheader("Summarized text with matched entities in Green and mismatched entities in Red relative to the Original Text")
489
+
490
+ st.write(entity_match_html, unsafe_allow_html=True)
491
+
492
+ st.markdown("####")
493
+
494
+ summary_downloader(summarized_text)
495
+
496
+
497
+ st.markdown("""
498
+ """)
499
+
500
+ st.markdown("![visitor badge](https://visitor-badge.glitch.me/badge?page_id=nickmuchi-article-text-summarizer)")
501
+ # In[ ]:
502
+
503
+
504
+
505
+