mrs83's picture
Update app.py
05a8bac verified
Raw History Blame Contribute Delete
14 kB
import gradio as gr
import requests
import argparse
import math
import json
import os
import torch
import traceback
from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList
class StringStoppingCriteria(StoppingCriteria):
def __init__(self, tokenizer, stop_strings):
self.tokenizer = tokenizer
self.stop_strings = stop_strings
def __call__(self, input_ids, scores, **kwargs):
generated_text = self.tokenizer.decode(input_ids[0], skip_special_tokens=False)
for stop_string in self.stop_strings:
if stop_string in generated_text[-(len(stop_string) + 20) :]:
if generated_text.strip().endswith(stop_string):
return True
return False
def create_demo(api_url, model_id=None):
# Standalone mode check
hf_model_id = model_id or os.environ.get("HF_MODEL_ID", "ethicalabs/Echo-DSRN-114M-Base")
local_model = None
local_tokenizer = None
model_metadata = None
if hf_model_id:
print(f"📦 Standalone Mode: Loading {hf_model_id} via Transformers...")
try:
local_tokenizer = AutoTokenizer.from_pretrained(hf_model_id, trust_remote_code=True)
local_model = AutoModelForCausalLM.from_pretrained(
hf_model_id,
trust_remote_code=True,
torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
device_map="auto",
)
local_model.eval()
print("✅ Model loaded successfully.")
# Calculate metadata
total_params = sum(p.numel() for p in local_model.parameters())
trainable_params = sum(p.numel() for p in local_model.parameters() if p.requires_grad)
config = local_model.config
model_metadata = {
"Total Parameters": f"{total_params:,}",
"Trainable Parameters": f"{trainable_params:,}",
"Layers": getattr(
config, "num_hidden_layers", getattr(config, "num_layers", "N/A")
),
"Hidden Size": getattr(config, "hidden_size", "N/A"),
"MLP Ratio": getattr(config, "mlp_ratio", "N/A"),
"Vocab Size": getattr(config, "vocab_size", "N/A"),
"Surprise Lambda Init": getattr(config, "surprise_lambda_init", "N/A"),
"Model Type": getattr(config, "model_type", "echo"),
}
except Exception as e:
print(f"❌ Failed to load model: {e}")
traceback.print_exc()
css = """
.token-container {
font-family: 'JetBrains Mono', 'Courier New', monospace;
line-height: 1.8;
padding: 24px;
background: #020617;
border-radius: 12px;
color: #e2e8f0;
min-height: 250px;
white-space: pre-wrap;
overflow-wrap: break-word;
border: 1px solid #1e293b;
position: relative;
overflow: visible !important;
}
.token {
display: inline-block;
padding: 0 1px;
border-radius: 3px;
position: relative;
cursor: help;
transition: all 0.2s;
}
.token:hover {
background: rgba(255, 255, 255, 0.15) !important;
z-index: 100;
}
.token-tooltip {
display: none;
position: absolute;
bottom: 130%;
left: 50%;
transform: translateX(-50%);
background: #1e293b;
color: white;
padding: 12px;
border-radius: 10px;
font-size: 0.85rem;
z-index: 1000;
white-space: nowrap;
box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.5);
border: 1px solid rgba(255, 255, 255, 0.2);
pointer-events: none;
text-align: center;
min-width: 120px;
}
.token:hover .token-tooltip {
display: block;
}
.mode-badge {
display: inline-block;
padding: 4px 12px;
border-radius: 20px;
font-size: 0.85rem;
font-weight: 600;
margin-bottom: 15px;
}
.mode-standalone { background: #059669; color: white; }
.mode-api { background: #4f46e5; color: white; }
.metadata-card {
background: var(--block-background-fill);
padding: 16px;
border-radius: 12px;
border: 1px solid var(--primary-200);
box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06);
}
:root.dark .metadata-card {
border-color: var(--primary-800);
}
.metadata-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(180px, 1fr));
gap: 16px;
}
.metadata-item {
font-size: 0.85rem;
color: var(--body-text-color-subdued);
display: flex;
flex-direction: column;
gap: 4px;
}
.metadata-value {
font-weight: 600;
color: var(--primary-600);
font-size: 1.05rem;
}
:root.dark .metadata-value {
color: var(--primary-400);
}
"""
def generate_html(prompt, max_tokens, temperature, top_k, rep_penalty):
try:
if local_model and local_tokenizer:
# Standalone Mode Logic
input_tokens = local_tokenizer(prompt, return_tensors="pt").to(local_model.device)
input_ids = input_tokens.input_ids
stop_strings = ["<|im_end|>", "<|end|>", "<|user|>"]
stopping_criteria = StoppingCriteriaList(
[StringStoppingCriteria(local_tokenizer, stop_strings)]
)
all_tokens = []
all_logprobs = []
# Echo (Prompt) Logprobs
with torch.no_grad():
outputs = local_model(input_ids)
logits = outputs.logits
for i in range(input_ids.shape[1]):
token_id = input_ids[0, i].item()
token_text = local_tokenizer.decode([token_id])
all_tokens.append(token_text)
if i == 0:
all_logprobs.append(None)
else:
lp = torch.nn.functional.log_softmax(logits[0, i - 1, :], dim=-1)
all_logprobs.append(lp[token_id].item())
# Generation
with torch.no_grad():
gen_out = local_model.generate(
**input_tokens,
max_new_tokens=max_tokens,
temperature=temperature,
top_k=top_k,
top_p=0.9,
repetition_penalty=rep_penalty,
do_sample=temperature > 0,
use_cache=False,
output_scores=True,
return_dict_in_generate=True,
stopping_criteria=stopping_criteria,
pad_token_id=local_tokenizer.pad_token_id or 32000,
eos_token_id=[32000, 32007, 32011],
)
output_ids = gen_out.sequences
scores = gen_out.scores
generated_ids = output_ids[0, input_ids.shape[1] :]
for i, token_id in enumerate(generated_ids):
token_id = token_id.item()
token_text = local_tokenizer.decode([token_id])
all_tokens.append(token_text)
lp = torch.nn.functional.log_softmax(scores[i][0, :], dim=-1)
all_logprobs.append(lp[token_id].item())
logprobs = {"tokens": all_tokens, "token_logprobs": all_logprobs}
else:
# API Mode
api_payload = {
"prompt": prompt,
"max_tokens": max_tokens,
"temperature": temperature,
"top_k": top_k,
"repetition_penalty": rep_penalty,
"logprobs": 1,
"echo": True,
}
response = requests.post(f"{api_url}/v1/completions", json=api_payload)
response.raise_for_status()
data = response.json()
logprobs = data["choices"][0]["logprobs"]
html = '<div class="token-container">'
for i, token in enumerate(logprobs["tokens"]):
lp = logprobs["token_logprobs"][i]
prob = math.exp(lp) if lp is not None else 1.0
# Sanitize token for HTML display
display_token = (
token.replace("\n", "↵\n")
.replace(" ", "&nbsp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
)
color = "#e2e8f0" # Default
if lp is not None:
if prob > 0.8:
color = "#4ade80" # Vibrant Green
elif prob < 0.2:
color = "#f87171" # Soft Red
prob_pct = f"{(prob * 100):.1f}%" if lp is not None else "N/A"
logit_str = f"{lp:.4f}" if lp is not None else "N/A"
html += f'<span class="token" style="color: {color}">{display_token}'
html += f'<span class="token-tooltip">'
html += f'<span style="opacity: 0.7; font-size: 0.7rem">CONFIDENCE</span><br>'
html += f'<span style="font-size: 1.1rem; font-weight: 800; color: {color}">{prob_pct}</span><br>'
html += f'<hr style="margin: 5px 0; opacity: 0.1; border: none; border-top: 1px solid white;">'
html += f'<span style="opacity: 0.7; font-size: 0.7rem">LOGIT</span><br>'
html += f'<span style="font-family: monospace">{logit_str}</span>'
html += f'</span></span>'
html += "</div>"
return html
except Exception as e:
traceback.print_exc()
return f'<div class="token-container" style="color: #f87171">❌ Error: {str(e)}</div>'
with gr.Blocks(title="Echo-DSRN-114M-Base - Next Word Prediction") as demo:
with gr.Row():
with gr.Column(scale=4):
gr.Markdown(
"""
# 🌫️ Echo-DSRN-114M-Base - Next Word Prediction
The **Echo-DSRN(N)** (**Dual State Recurrent Neural Network**, short name: `Echo-DSRN`, also know as `echo`) is a novel architecture specifically designed to be a viable alternative for low-resource tasks that are currently being inefficiently handled by the excessive scale of Large Language Models (LLMs)
"""
)
with gr.Column(scale=1):
mode_str = "Standalone Mode" if local_model else f"Connected to API: {api_url}"
mode_cls = "mode-standalone" if local_model else "mode-api"
gr.HTML(f'<div class="mode-badge {mode_cls}">{mode_str}</div>')
if model_metadata:
gr.Markdown("### 📦 Model Architecture Details")
metadata_html = '<div class="metadata-card"><div class="metadata-grid">'
for key, value in model_metadata.items():
metadata_html += f'<div class="metadata-item">{key}<span class="metadata-value">{value}</span></div>'
metadata_html += "</div></div>"
gr.HTML(metadata_html)
gr.Markdown(
"""
Hover over generated tokens to visualize the model's confidence and raw logit values.
"""
)
with gr.Row():
with gr.Column(scale=1):
prompt = gr.Textbox(
label="Input Prompt", placeholder="The capital of France is...", lines=5
)
with gr.Row():
max_tokens = gr.Slider(
minimum=1, maximum=512, value=128, step=1, label="Max Tokens"
)
temp = gr.Slider(
minimum=0.0, maximum=2.0, value=0.7, step=0.1, label="Temperature"
)
with gr.Row():
top_k = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top K")
rep_penalty = gr.Slider(
minimum=1.0, maximum=2.0, value=1.2, step=0.05, label="Repetition Penalty"
)
submit_btn = gr.Button("Generate with Probabilities", variant="primary")
with gr.Column(scale=1):
gr.Markdown("### Prediction Visualization")
output_html = gr.HTML(label="Visualized Output")
submit_btn.click(
generate_html,
inputs=[prompt, max_tokens, temp, top_k, rep_penalty],
outputs=output_html,
)
gr.Examples(
examples=[
["The capital of France is", 128, 0.1, 40, 1.2],
["Responsible AI development requires", 128, 0.7, 40, 1.2],
["To be or not to be, that is the", 128, 0.1, 40, 1.2],
],
inputs=[prompt, max_tokens, temp, top_k, rep_penalty],
)
return demo, css
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Echo-DSRN Next Word Prediction App")
parser.add_argument(
"--api_url", type=str, default="http://localhost:5000", help="URL of the Echo-DSRN API"
)
parser.add_argument(
"--model_id",
type=str,
default=None,
help="Local path or HF ID to the model for Standalone mode",
)
parser.add_argument("--port", type=int, default=7860, help="Port to run the Gradio app on")
args = parser.parse_args()
demo, css = create_demo(args.api_url, args.model_id)
demo.launch(server_port=args.port, css=css)