Spaces:
Running on Zero
Running on Zero
Download app.py from teamup-tech/MuScriptor: direct link, hf CLI and curl.
- Browser
- Download file 9.57 kB
-
https://huggingface.co/spaces/teamup-tech/MuScriptor/resolve/main/app.py
- Command line
-
hf download hf://spaces/teamup-tech/MuScriptor/app.py
-
curl -L -o app.py https://huggingface.co/spaces/teamup-tech/MuScriptor/resolve/main/app.py
9.57 kB
| import spaces # imported first, as ZeroGPU requires | |
| import sys | |
| sys.stdout.reconfigure(line_buffering=True) | |
| import contextlib | |
| import tempfile | |
| from pathlib import Path | |
| import gradio as gr | |
| import soundfile as sf | |
| import torch | |
| from huggingface_hub import hf_hub_download, snapshot_download | |
| from pyharp import ModelCard, build_endpoint | |
| from pyharp.tags import Subcategory | |
| from muscriptor import TranscriptionModel | |
| from muscriptor.tokenizer.mt3 import MT3_FULL_PLUS_GROUP_NAMES, resolve_instrument_names | |
| model_card = ModelCard( | |
| name="MuScriptor", | |
| description=( | |
| "Multi-instrument audio-to-MIDI transcription, trained on 170k songs " | |
| "from classical music to heavy metal. The code is MIT, but the " | |
| "published weights are CC-BY-NC-4.0, so results cannot be used " | |
| "commercially. Repository: https://github.com/muscriptor/muscriptor " | |
| "Paper: https://arxiv.org/abs/2607.08168" | |
| ), | |
| author="Simon Rouard, Michael Krause, Axel Roebel, Carl-Johann Simon-Gabriel, Alexandre Défossez (Kyutai x Mirelo)", | |
| tags=[Subcategory.MUSIC_ANALYSIS, 'multi-instrument', 'domain:music'], | |
| ) | |
| _VALID_INSTRUMENTS = ", ".join(MT3_FULL_PLUS_GROUP_NAMES) | |
| DEVICE = "cuda" if torch.cuda.is_available() else "cpu" | |
| # Each size's weights, pinned, with the config.json that load_model reads beside them | |
| _WEIGHT_REVISIONS = { | |
| "small": "8c127f603b807520fa465c838e9bfee8a91ada4e", | |
| "medium": "f32236969308476e01fd3aae67357de5feb05a2d", | |
| "large": "8809fdfbed2affa7ade94a7059e746e3880720e7", | |
| } | |
| _weights: dict[str, Path] = {} | |
| _models: dict[str, TranscriptionModel] = {} | |
| DEFAULT_SIZE = "medium" | |
| def _get_model(variant: str) -> TranscriptionModel: | |
| """Builds the TranscriptionModel of one size, on the GPU the call holds. | |
| Built inside the call, so ZeroGPU builds it again on every call, which takes seconds from local | |
| weights. It cannot be built in the worker at startup, since muscriptor loads its safetensors | |
| straight onto CUDA, which ZeroGPU's emulation does not answer outside a call, and its | |
| conditioners keep the device they were built for, so building on the CPU and moving it is not | |
| enough either. | |
| """ | |
| if variant not in _models: | |
| _models[variant] = TranscriptionModel.load_model(_weights[variant], device=DEVICE) | |
| return _models[variant] | |
| # Fetched in the worker, which runs process_fn and starts as the server comes up, so that no | |
| # request downloads inside its GPU reservation. The server never runs process_fn. | |
| if __name__ != "__main__": | |
| for _size, _revision in _WEIGHT_REVISIONS.items(): | |
| _weights[_size] = Path(snapshot_download( | |
| f"MuScriptor/muscriptor-{_size}", revision=_revision | |
| )) / "model.safetensors" | |
| # Tempo detection runs beat_this, which downloads its checkpoint from the JKU cloud on first | |
| # use, and that has failed both inside a GPU call (no name resolution) and at startup (503). | |
| # The same file is mirrored on the Hub and linked where torch.hub looks before downloading. | |
| _beat_checkpoint = Path(torch.hub.get_dir()) / "checkpoints" / "beat_this-final0.ckpt" | |
| if not _beat_checkpoint.exists(): | |
| _beat_checkpoint.parent.mkdir(parents=True, exist_ok=True) | |
| with contextlib.suppress(FileExistsError): | |
| _beat_checkpoint.symlink_to(hf_hub_download( | |
| "teamup-tech/Beat-This-weights", "final0.ckpt", | |
| revision="4afbe376535aaab71b76ae2ca2d7c16b08877321", | |
| )) | |
| def _resolve_instruments(text: str) -> tuple[list[str] | None, str]: | |
| """Resolve the comma-separated instruments box into exact group names. | |
| Names that don't resolve are dropped rather than raised: HARP's client | |
| shows generic error message, so an error here would not reach the user. | |
| Second value returns what happened as a .txt output | |
| """ | |
| tokens = [t for t in text.split(",") if t.strip()] | |
| if not tokens: | |
| return None, "No instrument restriction requested." | |
| resolved: list[str] = [] | |
| problems: list[str] = [] | |
| for token in tokens: | |
| try: | |
| resolved.extend(resolve_instrument_names([token])) | |
| except ValueError as e: | |
| problems.append(str(e)) | |
| if not problems: | |
| return resolved, f"Matched: {', '.join(resolved)}." | |
| note = "; ".join(problems) | |
| if resolved: | |
| note = f"Matched: {', '.join(resolved)}. Ignored: {note}" | |
| else: | |
| note = f"No valid instrument names found, transcribing unrestricted. {note}" | |
| return (resolved or None), note | |
| # The audio is transcribed in five-second segments, so the cost grows with its length, and with the | |
| # size of the model. Measured at 9 s for both 20 s and 60 s of audio on medium, which batches the | |
| # segments, and at 7 s and 14 s on large, and at 58 s for 600 s on medium, padded | |
| # PyHARP stops a job that runs past its timeout_s, 15 minutes by default, so a request whose | |
| # reservation would exceed that is refused rather than started. A shorter one is left to | |
| # ZeroGPU, which admits it when the caller's own allowance covers it. | |
| MAX_GPU_SECONDS = 900 | |
| SECONDS_PER_INPUT_SECOND = {"small": 0.1, "medium": 0.15, "large": 0.3} | |
| def gpu_seconds(input_audio_path, variant, instruments_text, use_sampling, temperature): | |
| return 20 + SECONDS_PER_INPUT_SECOND[variant] * sf.info(input_audio_path).duration | |
| def process_fn( | |
| input_audio_path: str, | |
| variant: str, | |
| instruments_text: str, | |
| use_sampling: bool, | |
| temperature: float, | |
| ) -> tuple[str, str]: | |
| """Transcribe the input audio to MIDI, plus a note on the Instruments field.""" | |
| # Checked before the GPU is reserved, so a request the model cannot take spends none of the | |
| # caller's ZeroGPU allowance | |
| if variant not in _WEIGHT_REVISIONS: | |
| raise gr.Error(f"Unknown model size: {variant}") | |
| if not input_audio_path: | |
| raise gr.Error("Provide the music to transcribe.") | |
| try: | |
| duration = sf.info(input_audio_path).duration | |
| except Exception as exc: | |
| raise gr.Error(f"Could not read the audio file: {exc}") from exc | |
| if duration == 0: | |
| raise gr.Error("The audio file is empty.") | |
| if gpu_seconds(input_audio_path, variant, instruments_text, use_sampling, | |
| temperature) > MAX_GPU_SECONDS: | |
| raise gr.Error( | |
| f"The music is {duration:.0f} s long, more than one request can transcribe in the " | |
| f"{MAX_GPU_SECONDS // 60} minutes it may run. Trim it, or use a smaller model." | |
| ) | |
| return transcribe(input_audio_path, variant, instruments_text, use_sampling, temperature) | |
| def transcribe(input_audio_path, variant, instruments_text, use_sampling, temperature): | |
| model = _get_model(variant) | |
| instruments, instrument_note = _resolve_instruments(instruments_text) | |
| midi_bytes = model.transcribe_to_midi( | |
| input_audio_path, | |
| use_sampling=use_sampling, | |
| temperature=temperature, | |
| instruments=instruments, | |
| ) | |
| with tempfile.NamedTemporaryFile(suffix=".mid", delete=False) as f: | |
| f.write(midi_bytes) | |
| output_midi_path = f.name | |
| notes_text = f"{Path(input_audio_path).name}\n\nInstrument Matching\n{instrument_note}\n" | |
| with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: | |
| f.write(notes_text) | |
| output_notes_path = f.name | |
| return output_midi_path, output_notes_path | |
| if __name__ == "__main__": | |
| with gr.Blocks() as demo: | |
| input_components = [ | |
| gr.Audio(type="filepath", label="Input Audio").harp_required(True).set_info(( | |
| "Music to transcribe. Trained on a wide range of material, from " | |
| "classical to metal, and it names the instruments it hears." | |
| )), | |
| gr.Dropdown( | |
| choices=["small", "medium", "large"], | |
| value=DEFAULT_SIZE, | |
| label="Model Size", | |
| info="small = fastest, least accurate. medium = balanced (default, per repo). large = most accurate, slower.", | |
| ), | |
| gr.Textbox( | |
| value="", | |
| label="Instruments", | |
| info=f"Separate names with commas. Leave blank to let the model detect instruments on its own. Valid names: {_VALID_INSTRUMENTS}.", | |
| ), | |
| gr.Checkbox( | |
| label="Use Sampling", | |
| info="Temperature sampling instead of greedy decoding (default: False, per repo).", | |
| ), | |
| gr.Slider( | |
| minimum=0.1, | |
| maximum=2.0, | |
| step=0.1, | |
| value=1.0, | |
| label="Temperature", | |
| info="Only used when Use Sampling is on. Higher values = note choices are more varied and unpredictable. Lower values = values are closer to model's most confident guesses. (default: 1.0, per repo).", | |
| ), | |
| ] | |
| output_components = [ | |
| gr.File(type="filepath", file_types=[".mid", ".midi"], label="Transcription").set_info( | |
| "Transcribed MIDI notes." | |
| ), | |
| gr.File(type="filepath", file_types=[".txt"], label="Instrument Matching").set_info( | |
| "Which requested instrument names were matched or ignored." | |
| ), | |
| ] | |
| build_endpoint( | |
| model_card=model_card, | |
| input_components=input_components, | |
| output_components=output_components, | |
| process_fn=process_fn, | |
| ) | |
| demo.queue().launch(show_error=True, pwa=True, ssr_mode=False) | |