app.py

import pysbd
from txtai.embeddings import Embeddings
import networkx as nx
from tqdm import tqdm
from txtai.graph import GraphFactory
from datasets import load_dataset
import streamlit as st
import streamlit.components.v1 as components
import string


st.set_page_config(page_title="DebateKG")
st.title("DebateKG - Automatic Policy Debate Case Creation")
st.caption("github: https://github.com/Hellisotherpeople/DebateKG")


form = st.sidebar.form("Main Settings")
form.header("Main Settings")
highlight_threshold = form.number_input("Enter the minimum similarity value needed to highlight" , value = 0.05)
show_extract = form.checkbox("Show extracts", value = True)
show_abstract = form.checkbox("Show abstract", value = False)
show_full_doc = form.checkbox("Show full doc", value = False)
show_citation = form.checkbox("Show citation", value = True)
rerank_word = form.text_input("(Optional) Constrain all evidence in the case to have this word within its text", value = "")
form.caption("Doing this may create graphs which are so constrained that DebateKG can't find a valid path in the graph to build a case")
html_window_width = form.number_input("Enter the pixel width of the output debate case window", value = 1000)
html_window_height = form.number_input("Enter the pixel height of the output debate case window", value = 1000)
option = form.selectbox(
 'Which Knowledge Graph do you want to use?',
 ('DebateSum_SemanticGraph_longformer_extract.tar.gz', 'DebateSum_SemanticGraph_longformer_abstract.tar.gz', 'DebateSum_SemanticGraph_mpnet_abstract.tar.gz', 'DebateSum_SemanticGraph_legalbert_abstract.tar.gz', 'DebateSum_SemanticGraph_legalbert_extract.tar.gz', 'DebateSum_SemanticGraph_mpnet_extract.tar.gz', 'DebateSum_SemanticGraph_mpnet_sentence.tar.gz'), index = 2)

form.form_submit_button("Change Settings")

@st.cache(allow_output_mutation=True)
def load_my_dataset():
 dataset = load_dataset("Hellisotherpeople/DebateSum", split = "train")
 return dataset

@st.cache(allow_output_mutation=True)
def load_embeddings():
 embeddings = Embeddings({
 "path": "sentence-transformers/all-mpnet-base-v2",
 "content": True,
 "functions": [
 {"name": "graph", "function": "graph.attribute"},
 ],
 "expressions": [
 {"name": "topic", "expression": "graph(indexid, 'topic')"},
 {"name": "topicrank", "expression": "graph(indexid, 'topicrank')"}
 ],
 "graph": {
 "limit": 100,
 "minscore": 0.10,
 "topics": {
 "terms": 4,
 "resolution" : 100
 }
 }
 })
 embeddings.load(option)
 return embeddings

dataset = load_my_dataset()
embeddings = load_embeddings()

graph = embeddings.graph

def david_distance(source, target, attrs):
 distance = max(1.0 - attrs["weight"], 0.0)
 return distance if distance >= 0.15 else 1.00

def david_showpath(source, target, the_graph):
 return nx.shortest_path(the_graph, source, target, david_distance)

def david_show_all_paths(source, target, the_graph):
 return nx.all_shortest_paths(the_graph, source, target, david_distance)


def highlight(index, result):
 output = f"{index}. "
 spans = [(token, score, "#fff59d" if score > highlight_threshold else None) for token, score in result["tokens"]]

 for token, _, color in spans:
 output += f"<span style='background-color: {color}'>{token}</span> " if color else f"{token} "

 return output



def showpath_any(list_of_arguments, strip_punctuation = True, the_graph=graph.backend):
 list_of_paths = []
 for x, y in zip(list_of_arguments, list_of_arguments[1:]):
 a_path = david_showpath(x, y, the_graph)
 list_of_paths.extend(a_path)
 #print(list_of_paths)
 path = [graph.attribute(p, "text") for p in list_of_paths]
 list_of_evidence_ids = []
 for text in path:
 if strip_punctuation:
 text = text.translate(str.maketrans("","", string.punctuation))
 list_of_evidence_ids.append(int(embeddings.search(f"select id from txtai where similar('{text}') limit 1")[0]['id']))

 sections = []
 #sections.append(list_of_evidence_ids)
 for x, p in enumerate(path):
 if x == 0:
 # Print start node
 
 sections.append(f"{x + 1}. {p}")
 if show_abstract:
 sections.append(dataset["Abstract"][list_of_evidence_ids[x]])
 if show_citation:
 sections.append(dataset["Citation"][list_of_evidence_ids[x]])
 if show_extract:
 sections.append(dataset["Extract"][list_of_evidence_ids[x]])
 if show_full_doc:
 sections.append(dataset["Full-Document"][list_of_evidence_ids[x]])

 if x < len(path) - 1:
 # Explain and highlight next path element
 results = embeddings.explain(p, [path[x + 1]], limit=1)[0]
 sections.append(highlight(x + 2, results))
 if show_abstract:
 sections.append(dataset["Abstract"][list_of_evidence_ids[x+1]])
 if show_citation:
 sections.append(dataset["Citation"][list_of_evidence_ids[x+1]])
 if show_extract:
 sections.append(dataset["Extract"][list_of_evidence_ids[x+1]])
 if show_full_doc:
 sections.append(dataset["Full-Document"][list_of_evidence_ids[x+1]])

 return components.html("<br/><br/>".join(sections), scrolling = True, width = html_window_width, height = html_window_height)

def question(text, rerank_word = "", rerank_topic = "", limit = 100):
 return embeddings.search(f"select id, text, topic, evidence_id, score from txtai where similar('{text}') and text like '%{rerank_word}%' and topic like '%{rerank_topic}%' limit {limit}")



query_form = st.form("Query the Index:")
query_form.write("Step 1: Find Arguments")
query_form.write("Use semantic SQL from txtai to find some arguments, we use indexids to keep track of them.")
query_form.caption("You can use the semantic SQL to explore the dataset too! The possibilities are limitless!")
query_sql = query_form.text_area("Enter a semantic SQL statement", value = f"select topic, * from txtai where similar('Trump and US relations with China') and topic like '%trump%' and text like '%china%' limit 1")

query_form_submitted = query_form.form_submit_button("Query")

if query_form_submitted:
 with st.expander("Output (Open Me)", expanded = False):
 #my_path = showpath_any([170750, 50, 23])
 #st.write(embeddings.search(f"select * from txtai where similar('you') and text like '%the%' limit 10"))
 st.write(embeddings.search(query_sql))


paths_form = st.form("Build the Arguments")
paths_form.write("Step 2: Build a Policy Debate Case")
paths_form.write("Enter any number of indexids (arguments), DebateKG will build a debate case out of it which links them all together")
user_paths_string = paths_form.text_area("Enter a list of indexids seperated by whitespace", value = "250 10000 2405")
user_paths_list_of_strings = user_paths_string.split()
user_paths_list = list(map(int, user_paths_list_of_strings)) 

paths_form_submitted = paths_form.form_submit_button("Build a Policy Debate Case")

if paths_form_submitted:
 if rerank_word:
 selected_nodes = [n for n,v in graph.backend.nodes(data=True) if rerank_word in v['text']] ##also works for topic
 H = graph.backend.subgraph(selected_nodes)
 showpath_any(user_paths_list, the_graph = H)
 else:
 showpath_any(user_paths_list)