import spaces # imported first, as ZeroGPU requires import sys sys.stdout.reconfigure(line_buffering=True) import contextlib import tempfile from pathlib import Path import gradio as gr import soundfile as sf import torch from huggingface_hub import hf_hub_download, snapshot_download from pyharp import ModelCard, build_endpoint from pyharp.tags import Subcategory from muscriptor import TranscriptionModel from muscriptor.tokenizer.mt3 import MT3_FULL_PLUS_GROUP_NAMES, resolve_instrument_names model_card = ModelCard( name="MuScriptor", description=( "Multi-instrument audio-to-MIDI transcription, trained on 170k songs " "from classical music to heavy metal. The code is MIT, but the " "published weights are CC-BY-NC-4.0, so results cannot be used " "commercially. Repository: https://github.com/muscriptor/muscriptor " "Paper: https://arxiv.org/abs/2607.08168" ), author="Simon Rouard, Michael Krause, Axel Roebel, Carl-Johann Simon-Gabriel, Alexandre Défossez (Kyutai x Mirelo)", tags=[Subcategory.MUSIC_ANALYSIS, 'multi-instrument', 'domain:music'], ) _VALID_INSTRUMENTS = ", ".join(MT3_FULL_PLUS_GROUP_NAMES) DEVICE = "cuda" if torch.cuda.is_available() else "cpu" # Each size's weights, pinned, with the config.json that load_model reads beside them _WEIGHT_REVISIONS = { "small": "8c127f603b807520fa465c838e9bfee8a91ada4e", "medium": "f32236969308476e01fd3aae67357de5feb05a2d", "large": "8809fdfbed2affa7ade94a7059e746e3880720e7", } _weights: dict[str, Path] = {} _models: dict[str, TranscriptionModel] = {} DEFAULT_SIZE = "medium" def _get_model(variant: str) -> TranscriptionModel: """Builds the TranscriptionModel of one size, on the GPU the call holds. Built inside the call, so ZeroGPU builds it again on every call, which takes seconds from local weights. It cannot be built in the worker at startup, since muscriptor loads its safetensors straight onto CUDA, which ZeroGPU's emulation does not answer outside a call, and its conditioners keep the device they were built for, so building on the CPU and moving it is not enough either. """ if variant not in _models: _models[variant] = TranscriptionModel.load_model(_weights[variant], device=DEVICE) return _models[variant] # Fetched in the worker, which runs process_fn and starts as the server comes up, so that no # request downloads inside its GPU reservation. The server never runs process_fn. if __name__ != "__main__": for _size, _revision in _WEIGHT_REVISIONS.items(): _weights[_size] = Path(snapshot_download( f"MuScriptor/muscriptor-{_size}", revision=_revision )) / "model.safetensors" # Tempo detection runs beat_this, which downloads its checkpoint from the JKU cloud on first # use, and that has failed both inside a GPU call (no name resolution) and at startup (503). # The same file is mirrored on the Hub and linked where torch.hub looks before downloading. _beat_checkpoint = Path(torch.hub.get_dir()) / "checkpoints" / "beat_this-final0.ckpt" if not _beat_checkpoint.exists(): _beat_checkpoint.parent.mkdir(parents=True, exist_ok=True) with contextlib.suppress(FileExistsError): _beat_checkpoint.symlink_to(hf_hub_download( "teamup-tech/Beat-This-weights", "final0.ckpt", revision="4afbe376535aaab71b76ae2ca2d7c16b08877321", )) def _resolve_instruments(text: str) -> tuple[list[str] | None, str]: """Resolve the comma-separated instruments box into exact group names. Names that don't resolve are dropped rather than raised: HARP's client shows generic error message, so an error here would not reach the user. Second value returns what happened as a .txt output """ tokens = [t for t in text.split(",") if t.strip()] if not tokens: return None, "No instrument restriction requested." resolved: list[str] = [] problems: list[str] = [] for token in tokens: try: resolved.extend(resolve_instrument_names([token])) except ValueError as e: problems.append(str(e)) if not problems: return resolved, f"Matched: {', '.join(resolved)}." note = "; ".join(problems) if resolved: note = f"Matched: {', '.join(resolved)}. Ignored: {note}" else: note = f"No valid instrument names found, transcribing unrestricted. {note}" return (resolved or None), note # The audio is transcribed in five-second segments, so the cost grows with its length, and with the # size of the model. Measured at 9 s for both 20 s and 60 s of audio on medium, which batches the # segments, and at 7 s and 14 s on large, and at 58 s for 600 s on medium, padded # PyHARP stops a job that runs past its timeout_s, 15 minutes by default, so a request whose # reservation would exceed that is refused rather than started. A shorter one is left to # ZeroGPU, which admits it when the caller's own allowance covers it. MAX_GPU_SECONDS = 900 SECONDS_PER_INPUT_SECOND = {"small": 0.1, "medium": 0.15, "large": 0.3} def gpu_seconds(input_audio_path, variant, instruments_text, use_sampling, temperature): return 20 + SECONDS_PER_INPUT_SECOND[variant] * sf.info(input_audio_path).duration def process_fn( input_audio_path: str, variant: str, instruments_text: str, use_sampling: bool, temperature: float, ) -> tuple[str, str]: """Transcribe the input audio to MIDI, plus a note on the Instruments field.""" # Checked before the GPU is reserved, so a request the model cannot take spends none of the # caller's ZeroGPU allowance if variant not in _WEIGHT_REVISIONS: raise gr.Error(f"Unknown model size: {variant}") if not input_audio_path: raise gr.Error("Provide the music to transcribe.") try: duration = sf.info(input_audio_path).duration except Exception as exc: raise gr.Error(f"Could not read the audio file: {exc}") from exc if duration == 0: raise gr.Error("The audio file is empty.") if gpu_seconds(input_audio_path, variant, instruments_text, use_sampling, temperature) > MAX_GPU_SECONDS: raise gr.Error( f"The music is {duration:.0f} s long, more than one request can transcribe in the " f"{MAX_GPU_SECONDS // 60} minutes it may run. Trim it, or use a smaller model." ) return transcribe(input_audio_path, variant, instruments_text, use_sampling, temperature) @spaces.GPU(duration=gpu_seconds) @torch.inference_mode() def transcribe(input_audio_path, variant, instruments_text, use_sampling, temperature): model = _get_model(variant) instruments, instrument_note = _resolve_instruments(instruments_text) midi_bytes = model.transcribe_to_midi( input_audio_path, use_sampling=use_sampling, temperature=temperature, instruments=instruments, ) with tempfile.NamedTemporaryFile(suffix=".mid", delete=False) as f: f.write(midi_bytes) output_midi_path = f.name notes_text = f"{Path(input_audio_path).name}\n\nInstrument Matching\n{instrument_note}\n" with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: f.write(notes_text) output_notes_path = f.name return output_midi_path, output_notes_path if __name__ == "__main__": with gr.Blocks() as demo: input_components = [ gr.Audio(type="filepath", label="Input Audio").harp_required(True).set_info(( "Music to transcribe. Trained on a wide range of material, from " "classical to metal, and it names the instruments it hears." )), gr.Dropdown( choices=["small", "medium", "large"], value=DEFAULT_SIZE, label="Model Size", info="small = fastest, least accurate. medium = balanced (default, per repo). large = most accurate, slower.", ), gr.Textbox( value="", label="Instruments", info=f"Separate names with commas. Leave blank to let the model detect instruments on its own. Valid names: {_VALID_INSTRUMENTS}.", ), gr.Checkbox( label="Use Sampling", info="Temperature sampling instead of greedy decoding (default: False, per repo).", ), gr.Slider( minimum=0.1, maximum=2.0, step=0.1, value=1.0, label="Temperature", info="Only used when Use Sampling is on. Higher values = note choices are more varied and unpredictable. Lower values = values are closer to model's most confident guesses. (default: 1.0, per repo).", ), ] output_components = [ gr.File(type="filepath", file_types=[".mid", ".midi"], label="Transcription").set_info( "Transcribed MIDI notes." ), gr.File(type="filepath", file_types=[".txt"], label="Instrument Matching").set_info( "Which requested instrument names were matched or ignored." ), ] build_endpoint( model_card=model_card, input_components=input_components, output_components=output_components, process_fn=process_fn, ) demo.queue().launch(show_error=True, pwa=True, ssr_mode=False)