import gradio as gr import requests import argparse import math import json import os import torch import traceback from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList class StringStoppingCriteria(StoppingCriteria): def __init__(self, tokenizer, stop_strings): self.tokenizer = tokenizer self.stop_strings = stop_strings def __call__(self, input_ids, scores, **kwargs): generated_text = self.tokenizer.decode(input_ids[0], skip_special_tokens=False) for stop_string in self.stop_strings: if stop_string in generated_text[-(len(stop_string) + 20) :]: if generated_text.strip().endswith(stop_string): return True return False def create_demo(api_url, model_id=None): # Standalone mode check hf_model_id = model_id or os.environ.get("HF_MODEL_ID", "ethicalabs/Echo-DSRN-114M-Base") local_model = None local_tokenizer = None model_metadata = None if hf_model_id: print(f"📦 Standalone Mode: Loading {hf_model_id} via Transformers...") try: local_tokenizer = AutoTokenizer.from_pretrained(hf_model_id, trust_remote_code=True) local_model = AutoModelForCausalLM.from_pretrained( hf_model_id, trust_remote_code=True, torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32, device_map="auto", ) local_model.eval() print("✅ Model loaded successfully.") # Calculate metadata total_params = sum(p.numel() for p in local_model.parameters()) trainable_params = sum(p.numel() for p in local_model.parameters() if p.requires_grad) config = local_model.config model_metadata = { "Total Parameters": f"{total_params:,}", "Trainable Parameters": f"{trainable_params:,}", "Layers": getattr( config, "num_hidden_layers", getattr(config, "num_layers", "N/A") ), "Hidden Size": getattr(config, "hidden_size", "N/A"), "MLP Ratio": getattr(config, "mlp_ratio", "N/A"), "Vocab Size": getattr(config, "vocab_size", "N/A"), "Surprise Lambda Init": getattr(config, "surprise_lambda_init", "N/A"), "Model Type": getattr(config, "model_type", "echo"), } except Exception as e: print(f"❌ Failed to load model: {e}") traceback.print_exc() css = """ .token-container { font-family: 'JetBrains Mono', 'Courier New', monospace; line-height: 1.8; padding: 24px; background: #020617; border-radius: 12px; color: #e2e8f0; min-height: 250px; white-space: pre-wrap; overflow-wrap: break-word; border: 1px solid #1e293b; position: relative; overflow: visible !important; } .token { display: inline-block; padding: 0 1px; border-radius: 3px; position: relative; cursor: help; transition: all 0.2s; } .token:hover { background: rgba(255, 255, 255, 0.15) !important; z-index: 100; } .token-tooltip { display: none; position: absolute; bottom: 130%; left: 50%; transform: translateX(-50%); background: #1e293b; color: white; padding: 12px; border-radius: 10px; font-size: 0.85rem; z-index: 1000; white-space: nowrap; box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.5); border: 1px solid rgba(255, 255, 255, 0.2); pointer-events: none; text-align: center; min-width: 120px; } .token:hover .token-tooltip { display: block; } .mode-badge { display: inline-block; padding: 4px 12px; border-radius: 20px; font-size: 0.85rem; font-weight: 600; margin-bottom: 15px; } .mode-standalone { background: #059669; color: white; } .mode-api { background: #4f46e5; color: white; } .metadata-card { background: var(--block-background-fill); padding: 16px; border-radius: 12px; border: 1px solid var(--primary-200); box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06); } :root.dark .metadata-card { border-color: var(--primary-800); } .metadata-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 16px; } .metadata-item { font-size: 0.85rem; color: var(--body-text-color-subdued); display: flex; flex-direction: column; gap: 4px; } .metadata-value { font-weight: 600; color: var(--primary-600); font-size: 1.05rem; } :root.dark .metadata-value { color: var(--primary-400); } """ def generate_html(prompt, max_tokens, temperature, top_k, rep_penalty): try: if local_model and local_tokenizer: # Standalone Mode Logic input_tokens = local_tokenizer(prompt, return_tensors="pt").to(local_model.device) input_ids = input_tokens.input_ids stop_strings = ["<|im_end|>", "<|end|>", "<|user|>"] stopping_criteria = StoppingCriteriaList( [StringStoppingCriteria(local_tokenizer, stop_strings)] ) all_tokens = [] all_logprobs = [] # Echo (Prompt) Logprobs with torch.no_grad(): outputs = local_model(input_ids) logits = outputs.logits for i in range(input_ids.shape[1]): token_id = input_ids[0, i].item() token_text = local_tokenizer.decode([token_id]) all_tokens.append(token_text) if i == 0: all_logprobs.append(None) else: lp = torch.nn.functional.log_softmax(logits[0, i - 1, :], dim=-1) all_logprobs.append(lp[token_id].item()) # Generation with torch.no_grad(): gen_out = local_model.generate( **input_tokens, max_new_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=0.9, repetition_penalty=rep_penalty, do_sample=temperature > 0, use_cache=False, output_scores=True, return_dict_in_generate=True, stopping_criteria=stopping_criteria, pad_token_id=local_tokenizer.pad_token_id or 32000, eos_token_id=[32000, 32007, 32011], ) output_ids = gen_out.sequences scores = gen_out.scores generated_ids = output_ids[0, input_ids.shape[1] :] for i, token_id in enumerate(generated_ids): token_id = token_id.item() token_text = local_tokenizer.decode([token_id]) all_tokens.append(token_text) lp = torch.nn.functional.log_softmax(scores[i][0, :], dim=-1) all_logprobs.append(lp[token_id].item()) logprobs = {"tokens": all_tokens, "token_logprobs": all_logprobs} else: # API Mode api_payload = { "prompt": prompt, "max_tokens": max_tokens, "temperature": temperature, "top_k": top_k, "repetition_penalty": rep_penalty, "logprobs": 1, "echo": True, } response = requests.post(f"{api_url}/v1/completions", json=api_payload) response.raise_for_status() data = response.json() logprobs = data["choices"][0]["logprobs"] html = '
' for i, token in enumerate(logprobs["tokens"]): lp = logprobs["token_logprobs"][i] prob = math.exp(lp) if lp is not None else 1.0 # Sanitize token for HTML display display_token = ( token.replace("\n", "↵\n") .replace(" ", " ") .replace("<", "<") .replace(">", ">") ) color = "#e2e8f0" # Default if lp is not None: if prob > 0.8: color = "#4ade80" # Vibrant Green elif prob < 0.2: color = "#f87171" # Soft Red prob_pct = f"{(prob * 100):.1f}%" if lp is not None else "N/A" logit_str = f"{lp:.4f}" if lp is not None else "N/A" html += f'{display_token}' html += f'' html += f'CONFIDENCE
' html += f'{prob_pct}
' html += f'
' html += f'LOGIT
' html += f'{logit_str}' html += f'
' html += "
" return html except Exception as e: traceback.print_exc() return f'
❌ Error: {str(e)}
' with gr.Blocks(title="Echo-DSRN-114M-Base - Next Word Prediction") as demo: with gr.Row(): with gr.Column(scale=4): gr.Markdown( """ # 🌫️ Echo-DSRN-114M-Base - Next Word Prediction The **Echo-DSRN(N)** (**Dual State Recurrent Neural Network**, short name: `Echo-DSRN`, also know as `echo`) is a novel architecture specifically designed to be a viable alternative for low-resource tasks that are currently being inefficiently handled by the excessive scale of Large Language Models (LLMs) """ ) with gr.Column(scale=1): mode_str = "Standalone Mode" if local_model else f"Connected to API: {api_url}" mode_cls = "mode-standalone" if local_model else "mode-api" gr.HTML(f'
{mode_str}
') if model_metadata: gr.Markdown("### 📦 Model Architecture Details") metadata_html = '
' for key, value in model_metadata.items(): metadata_html += f'' metadata_html += "
" gr.HTML(metadata_html) gr.Markdown( """ Hover over generated tokens to visualize the model's confidence and raw logit values. """ ) with gr.Row(): with gr.Column(scale=1): prompt = gr.Textbox( label="Input Prompt", placeholder="The capital of France is...", lines=5 ) with gr.Row(): max_tokens = gr.Slider( minimum=1, maximum=512, value=128, step=1, label="Max Tokens" ) temp = gr.Slider( minimum=0.0, maximum=2.0, value=0.7, step=0.1, label="Temperature" ) with gr.Row(): top_k = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top K") rep_penalty = gr.Slider( minimum=1.0, maximum=2.0, value=1.2, step=0.05, label="Repetition Penalty" ) submit_btn = gr.Button("Generate with Probabilities", variant="primary") with gr.Column(scale=1): gr.Markdown("### Prediction Visualization") output_html = gr.HTML(label="Visualized Output") submit_btn.click( generate_html, inputs=[prompt, max_tokens, temp, top_k, rep_penalty], outputs=output_html, ) gr.Examples( examples=[ ["The capital of France is", 128, 0.1, 40, 1.2], ["Responsible AI development requires", 128, 0.7, 40, 1.2], ["To be or not to be, that is the", 128, 0.1, 40, 1.2], ], inputs=[prompt, max_tokens, temp, top_k, rep_penalty], ) return demo, css if __name__ == "__main__": parser = argparse.ArgumentParser(description="Echo-DSRN Next Word Prediction App") parser.add_argument( "--api_url", type=str, default="http://localhost:5000", help="URL of the Echo-DSRN API" ) parser.add_argument( "--model_id", type=str, default=None, help="Local path or HF ID to the model for Standalone mode", ) parser.add_argument("--port", type=int, default=7860, help="Port to run the Gradio app on") args = parser.parse_args() demo, css = create_demo(args.api_url, args.model_id) demo.launch(server_port=args.port, css=css)