Post
577
Model is always disabled?
#script...
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2",
token="xxxxxx")
That loads the model fine. But if used by index returned from VectorStoreIndex for QDrant like this:
#script...
query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
response = query_engine.query(
"What is formula 1?"
)
response.print_response_stream()
It errors out with a disabled error:
AssertionError Traceback (most recent call last)
Cell In[34], line 1
----> 1 query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
3 response = query_engine.query(
4 "What is formula 1?"
5 )
7 response.print_response_stream()
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/indices/base.py:376, in BaseIndex.as_query_engine(self, llm, **kwargs)
370 from llama_index.core.query_engine.retriever_query_engine import (
371 RetrieverQueryEngine,
372 )
374 retriever = self.as_retriever(**kwargs)
375 llm = (
--> 376 resolve_llm(llm, callback_manager=self._callback_manager)
377 if llm
378 else Settings.llm
379 )
381 return RetrieverQueryEngine.from_args(
382 retriever,
383 llm=llm,
384 **kwargs,
385 )
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/llms/utils.py:102, in resolve_llm(llm, callback_manager)
99 print("LLM is explicitly disabled. Using MockLLM.")
100 llm = MockLLM()
--> 102 assert isinstance(llm, LLM)
104 llm.callback_manager = callback_manager or Settings.callback_manager
106 return llm
AssertionError:
So why is the LLM disabled?
Thanks!
#script...
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2",
token="xxxxxx")
That loads the model fine. But if used by index returned from VectorStoreIndex for QDrant like this:
#script...
query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
response = query_engine.query(
"What is formula 1?"
)
response.print_response_stream()
It errors out with a disabled error:
AssertionError Traceback (most recent call last)
Cell In[34], line 1
----> 1 query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
3 response = query_engine.query(
4 "What is formula 1?"
5 )
7 response.print_response_stream()
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/indices/base.py:376, in BaseIndex.as_query_engine(self, llm, **kwargs)
370 from llama_index.core.query_engine.retriever_query_engine import (
371 RetrieverQueryEngine,
372 )
374 retriever = self.as_retriever(**kwargs)
375 llm = (
--> 376 resolve_llm(llm, callback_manager=self._callback_manager)
377 if llm
378 else Settings.llm
379 )
381 return RetrieverQueryEngine.from_args(
382 retriever,
383 llm=llm,
384 **kwargs,
385 )
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/llms/utils.py:102, in resolve_llm(llm, callback_manager)
99 print("LLM is explicitly disabled. Using MockLLM.")
100 llm = MockLLM()
--> 102 assert isinstance(llm, LLM)
104 llm.callback_manager = callback_manager or Settings.callback_manager
106 return llm
AssertionError:
So why is the LLM disabled?
Thanks!