Update app.py

app.py

@@ -1,102 +1,86 @@
 import gradio as gr
-from transformers import pipeline,WhisperProcessor, WhisperForConditionalGeneration
+from transformers import AutoProcessor, BlipForConditionalGeneration, AutoModelForCausalLM, AutoImageProcessor, VisionEncoderDecoderModel, AutoTokenizer
+import io
+import base64
+
+# from transformers import AutoProcessor, AutoTokenizer, AutoImageProcessor, AutoModelForCausalLM, BlipForConditionalGeneration, Blip2ForConditionalGeneration, VisionEncoderDecoderModel
 import torch
-import librosa
-import datasets
-from transformers.pipelines.pt_utils import KeyDataset
-from tqdm.auto import tqdm
-import logging
-import time
-import uuid
-import soundfile as sf
-from PIL import Image
-
-# model.py apache license 2.0 Copyright      2022-2023  Xiaomi Corp.        (authors: Fangjun Kuang)
-from model import get_pretrained_model, language_to_models
-# demo for a input given image transform into text interpretation, and those text put a speech text to be played 
-
-#text to speech code from https://huggingface.co/spaces/k2-fsa/text-to-speech/blob/main/app.py
-image_to_text_model = pipeline("image-classification",model="microsoft/beit-base-patch16-224-pt22k-ft22k")
-
-def build_html_output(s: str, style: str = "result_item_success"):
-    return f"""
-    <div class='result'>
-        <div class='result_item {style}'>
-          {s}
-        </div>
-    </div>
-    """
-def text_to_speech(language: str, repo_id: str, text: str, sid: str, speed: float):
-    logging.info(f"Input text: {text}. sid: {sid}, speed: {speed}")
-    sid = int(sid)
-    tts = get_pretrained_model(repo_id, speed)
-
-    start = time.time()
-    audio = tts.generate(text, sid=sid)
-    end = time.time()
-
-    if len(audio.samples) == 0:
-        raise ValueError(
-            "Error in generating audios. Please read previous error messages."
-        )
-
-    duration = len(audio.samples) / audio.sample_rate
-
-    elapsed_seconds = end - start
-    rtf = elapsed_seconds / duration
-
-    info = f"""
-    Wave duration  : {duration:.3f} s <br/>
-    Processing time: {elapsed_seconds:.3f} s <br/>
-    RTF: {elapsed_seconds:.3f}/{duration:.3f} = {rtf:.3f} <br/>
-    """
-
-    logging.info(info)
-    logging.info(f"\nrepo_id: {repo_id}\ntext: {text}\nsid: {sid}\nspeed: {speed}")
-
-    filename = str(uuid.uuid4())
-    filename = f"{filename}.wav"
-    sf.write(
-        filename,
-        audio.samples,
-        samplerate=audio.sample_rate,
-        subtype="PCM_16",
-    )
-
-    return filename, build_html_output(info)
-
-
-with gr.Blocks() as demo:
-    language_choices = list(language_to_models.keys())
-    inputsImg=gr.Image()
-    idx=0
-    text_output = image_to_text_model(inputsImg)[0]
-    print(text_output)
-    for txt in text_output:        
-        output_txt[idx] = gr.Textbox(label=text_output,lines=1,max_lines=1,value=text_output,placeholder="Interpretation")
-        input_sid = gr.Textbox(
-                label="Speaker ID",
-                info="Speaker ID",
-                lines=1,
-                max_lines=1,
-                value="0",
-                placeholder="Speaker ID. Valid only for mult-speaker model")
-        
-        input_speed = gr.Slider(
-                minimum=0.1,
-                maximum=10,
-                value=1,
-                step=0.1,
-                label="Speed (larger->faster; smaller->slower)")
-        text_to_speech(language_choices[0],language_to_models[language_choices[0]][0],text_output,input_sid,input_speed)
-        output_audio[idx] = gr.Audio(label="Output")
-        output_info[idx] = gr.HTML(label="Info")
-        idx=idx+1
-    demo=gr.Interface(fn=text_to_speech, 
-                           title="Image to Text Interpretation",
-                           inputs=inputsImg,
-                           outputs=[output_txt,output_audio,input_sid,input_speed],
-                           description="image to audio demo",
-                           article = ""
-                           )
-demo.launch()
+import open_clip
+import openai
+
+from huggingface_hub import hf_hub_download
+
+# Carga el modelo de clasificación de imagen a texto
+blip_processor_large = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
+blip_model_large = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
+
+# Carga el modelo de texto a voz
+openai.api_key = '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'
+model_id = "base"
+#model_version = "2022-01-01"
+whisper = openai.Model(model_id=model_id)
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+blip_model_large.to(device)
+
+def generate_caption(processor, model, image, tokenizer=None, use_float_16=False):
+    inputs = processor(images=image, return_tensors="pt").to(device)
+
+    if use_float_16:
+        inputs = inputs.to(torch.float16)
+    
+    generated_ids = model.generate(pixel_values=inputs.pixel_values, max_length=50)
+
+    if tokenizer is not None:
+        generated_caption = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    else:
+        generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+   
+    return generated_caption
+
+
+def generate_caption_coca(model, transform, image):
+    im = transform(image).unsqueeze(0).to(device)
+    with torch.no_grad(), torch.cuda.amp.autocast():
+        generated = model.generate(im, seq_len=20)
+    return open_clip.decode(generated[0].detach()).split("<end_of_text>")[0].replace("<start_of_text>", "")
+
+
+def generate_captions(image):
+
+    caption_blip_large = generate_caption(blip_processor_large, blip_model_large, image)
+    print(caption_blip_large)
+    return caption_blip_large
+    
+
+# Define la función que convierte texto en voz
+def text_to_speech(text):
+    # Genera el audio utilizando el modelo Whisper
+    response = whisper.generate(prompt=text)
+    print(response)
+    # Extrae el audio del resultado
+    audio = response.choices[0].audio
+
+    # Codifica el audio en base64
+    audio_base64 = base64.b64encode(audio).decode("utf-8")
+
+    # Devuelve el audio como un archivo MP3
+    return BytesIO(base64.b64decode(audio_base64))
+
+# Define la interfaz de usuario utilizando Gradio
+inputsImg = [
+    gr.Image(type="pil", label="Imagen"),
+]
+
+outputs = [ gr.Textbox(label="Caption generated by BLIP-large") ] 
+title = "Clasificación de imagen a texto y conversión de texto a voz"
+description = "Carga una imagen y obtén una descripción de texto de lo que contiene la imagen, así como un archivo de audio que lee el texto en voz alta."
+examples = []
+
+interface = gr.Interface(fn=generate_captions, 
+                         inputs=inputsImg,
+                         outputs=outputs,
+                         examples=examples, 
+                         title=title,
+                         description=description)
+interface.launch()