import gradio as gr import requests import argparse import math import json import os import torch import traceback from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList class StringStoppingCriteria(StoppingCriteria): def __init__(self, tokenizer, stop_strings): self.tokenizer = tokenizer self.stop_strings = stop_strings def __call__(self, input_ids, scores, **kwargs): generated_text = self.tokenizer.decode(input_ids[0], skip_special_tokens=False) for stop_string in self.stop_strings: if stop_string in generated_text[-(len(stop_string) + 20) :]: if generated_text.strip().endswith(stop_string): return True return False def create_demo(api_url, model_id=None): # Standalone mode check hf_model_id = model_id or os.environ.get("HF_MODEL_ID", "ethicalabs/Echo-DSRN-114M-Base") local_model = None local_tokenizer = None model_metadata = None if hf_model_id: print(f"📦 Standalone Mode: Loading {hf_model_id} via Transformers...") try: local_tokenizer = AutoTokenizer.from_pretrained(hf_model_id, trust_remote_code=True) local_model = AutoModelForCausalLM.from_pretrained( hf_model_id, trust_remote_code=True, torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32, device_map="auto", ) local_model.eval() print("✅ Model loaded successfully.") # Calculate metadata total_params = sum(p.numel() for p in local_model.parameters()) trainable_params = sum(p.numel() for p in local_model.parameters() if p.requires_grad) config = local_model.config model_metadata = { "Total Parameters": f"{total_params:,}", "Trainable Parameters": f"{trainable_params:,}", "Layers": getattr( config, "num_hidden_layers", getattr(config, "num_layers", "N/A") ), "Hidden Size": getattr(config, "hidden_size", "N/A"), "MLP Ratio": getattr(config, "mlp_ratio", "N/A"), "Vocab Size": getattr(config, "vocab_size", "N/A"), "Surprise Lambda Init": getattr(config, "surprise_lambda_init", "N/A"), "Model Type": getattr(config, "model_type", "echo"), } except Exception as e: print(f"❌ Failed to load model: {e}") traceback.print_exc() css = """ .token-container { font-family: 'JetBrains Mono', 'Courier New', monospace; line-height: 1.8; padding: 24px; background: #020617; border-radius: 12px; color: #e2e8f0; min-height: 250px; white-space: pre-wrap; overflow-wrap: break-word; border: 1px solid #1e293b; position: relative; overflow: visible !important; } .token { display: inline-block; padding: 0 1px; border-radius: 3px; position: relative; cursor: help; transition: all 0.2s; } .token:hover { background: rgba(255, 255, 255, 0.15) !important; z-index: 100; } .token-tooltip { display: none; position: absolute; bottom: 130%; left: 50%; transform: translateX(-50%); background: #1e293b; color: white; padding: 12px; border-radius: 10px; font-size: 0.85rem; z-index: 1000; white-space: nowrap; box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.5); border: 1px solid rgba(255, 255, 255, 0.2); pointer-events: none; text-align: center; min-width: 120px; } .token:hover .token-tooltip { display: block; } .mode-badge { display: inline-block; padding: 4px 12px; border-radius: 20px; font-size: 0.85rem; font-weight: 600; margin-bottom: 15px; } .mode-standalone { background: #059669; color: white; } .mode-api { background: #4f46e5; color: white; } .metadata-card { background: var(--block-background-fill); padding: 16px; border-radius: 12px; border: 1px solid var(--primary-200); box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06); } :root.dark .metadata-card { border-color: var(--primary-800); } .metadata-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 16px; } .metadata-item { font-size: 0.85rem; color: var(--body-text-color-subdued); display: flex; flex-direction: column; gap: 4px; } .metadata-value { font-weight: 600; color: var(--primary-600); font-size: 1.05rem; } :root.dark .metadata-value { color: var(--primary-400); } """ def generate_html(prompt, max_tokens, temperature, top_k, rep_penalty): try: if local_model and local_tokenizer: # Standalone Mode Logic input_tokens = local_tokenizer(prompt, return_tensors="pt").to(local_model.device) input_ids = input_tokens.input_ids stop_strings = ["<|im_end|>", "<|end|>", "<|user|>"] stopping_criteria = StoppingCriteriaList( [StringStoppingCriteria(local_tokenizer, stop_strings)] ) all_tokens = [] all_logprobs = [] # Echo (Prompt) Logprobs with torch.no_grad(): outputs = local_model(input_ids) logits = outputs.logits for i in range(input_ids.shape[1]): token_id = input_ids[0, i].item() token_text = local_tokenizer.decode([token_id]) all_tokens.append(token_text) if i == 0: all_logprobs.append(None) else: lp = torch.nn.functional.log_softmax(logits[0, i - 1, :], dim=-1) all_logprobs.append(lp[token_id].item()) # Generation with torch.no_grad(): gen_out = local_model.generate( **input_tokens, max_new_tokens=max_tokens, temperature=temperature, top_k=top_k, top_p=0.9, repetition_penalty=rep_penalty, do_sample=temperature > 0, use_cache=False, output_scores=True, return_dict_in_generate=True, stopping_criteria=stopping_criteria, pad_token_id=local_tokenizer.pad_token_id or 32000, eos_token_id=[32000, 32007, 32011], ) output_ids = gen_out.sequences scores = gen_out.scores generated_ids = output_ids[0, input_ids.shape[1] :] for i, token_id in enumerate(generated_ids): token_id = token_id.item() token_text = local_tokenizer.decode([token_id]) all_tokens.append(token_text) lp = torch.nn.functional.log_softmax(scores[i][0, :], dim=-1) all_logprobs.append(lp[token_id].item()) logprobs = {"tokens": all_tokens, "token_logprobs": all_logprobs} else: # API Mode api_payload = { "prompt": prompt, "max_tokens": max_tokens, "temperature": temperature, "top_k": top_k, "repetition_penalty": rep_penalty, "logprobs": 1, "echo": True, } response = requests.post(f"{api_url}/v1/completions", json=api_payload) response.raise_for_status() data = response.json() logprobs = data["choices"][0]["logprobs"] html = '