MuScriptor / app.py
harp-dev's picture
Drop the comment on ssr_mode
3e602ad verified
Raw History Blame Contribute Delete
9.57 kB
import spaces # imported first, as ZeroGPU requires
import sys
sys.stdout.reconfigure(line_buffering=True)
import contextlib
import tempfile
from pathlib import Path
import gradio as gr
import soundfile as sf
import torch
from huggingface_hub import hf_hub_download, snapshot_download
from pyharp import ModelCard, build_endpoint
from pyharp.tags import Subcategory
from muscriptor import TranscriptionModel
from muscriptor.tokenizer.mt3 import MT3_FULL_PLUS_GROUP_NAMES, resolve_instrument_names
model_card = ModelCard(
name="MuScriptor",
description=(
"Multi-instrument audio-to-MIDI transcription, trained on 170k songs "
"from classical music to heavy metal. The code is MIT, but the "
"published weights are CC-BY-NC-4.0, so results cannot be used "
"commercially. Repository: https://github.com/muscriptor/muscriptor "
"Paper: https://arxiv.org/abs/2607.08168"
),
author="Simon Rouard, Michael Krause, Axel Roebel, Carl-Johann Simon-Gabriel, Alexandre Défossez (Kyutai x Mirelo)",
tags=[Subcategory.MUSIC_ANALYSIS, 'multi-instrument', 'domain:music'],
)
_VALID_INSTRUMENTS = ", ".join(MT3_FULL_PLUS_GROUP_NAMES)
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
# Each size's weights, pinned, with the config.json that load_model reads beside them
_WEIGHT_REVISIONS = {
"small": "8c127f603b807520fa465c838e9bfee8a91ada4e",
"medium": "f32236969308476e01fd3aae67357de5feb05a2d",
"large": "8809fdfbed2affa7ade94a7059e746e3880720e7",
}
_weights: dict[str, Path] = {}
_models: dict[str, TranscriptionModel] = {}
DEFAULT_SIZE = "medium"
def _get_model(variant: str) -> TranscriptionModel:
"""Builds the TranscriptionModel of one size, on the GPU the call holds.
Built inside the call, so ZeroGPU builds it again on every call, which takes seconds from local
weights. It cannot be built in the worker at startup, since muscriptor loads its safetensors
straight onto CUDA, which ZeroGPU's emulation does not answer outside a call, and its
conditioners keep the device they were built for, so building on the CPU and moving it is not
enough either.
"""
if variant not in _models:
_models[variant] = TranscriptionModel.load_model(_weights[variant], device=DEVICE)
return _models[variant]
# Fetched in the worker, which runs process_fn and starts as the server comes up, so that no
# request downloads inside its GPU reservation. The server never runs process_fn.
if __name__ != "__main__":
for _size, _revision in _WEIGHT_REVISIONS.items():
_weights[_size] = Path(snapshot_download(
f"MuScriptor/muscriptor-{_size}", revision=_revision
)) / "model.safetensors"
# Tempo detection runs beat_this, which downloads its checkpoint from the JKU cloud on first
# use, and that has failed both inside a GPU call (no name resolution) and at startup (503).
# The same file is mirrored on the Hub and linked where torch.hub looks before downloading.
_beat_checkpoint = Path(torch.hub.get_dir()) / "checkpoints" / "beat_this-final0.ckpt"
if not _beat_checkpoint.exists():
_beat_checkpoint.parent.mkdir(parents=True, exist_ok=True)
with contextlib.suppress(FileExistsError):
_beat_checkpoint.symlink_to(hf_hub_download(
"teamup-tech/Beat-This-weights", "final0.ckpt",
revision="4afbe376535aaab71b76ae2ca2d7c16b08877321",
))
def _resolve_instruments(text: str) -> tuple[list[str] | None, str]:
"""Resolve the comma-separated instruments box into exact group names.
Names that don't resolve are dropped rather than raised: HARP's client
shows generic error message, so an error here would not reach the user.
Second value returns what happened as a .txt output
"""
tokens = [t for t in text.split(",") if t.strip()]
if not tokens:
return None, "No instrument restriction requested."
resolved: list[str] = []
problems: list[str] = []
for token in tokens:
try:
resolved.extend(resolve_instrument_names([token]))
except ValueError as e:
problems.append(str(e))
if not problems:
return resolved, f"Matched: {', '.join(resolved)}."
note = "; ".join(problems)
if resolved:
note = f"Matched: {', '.join(resolved)}. Ignored: {note}"
else:
note = f"No valid instrument names found, transcribing unrestricted. {note}"
return (resolved or None), note
# The audio is transcribed in five-second segments, so the cost grows with its length, and with the
# size of the model. Measured at 9 s for both 20 s and 60 s of audio on medium, which batches the
# segments, and at 7 s and 14 s on large, and at 58 s for 600 s on medium, padded
# PyHARP stops a job that runs past its timeout_s, 15 minutes by default, so a request whose
# reservation would exceed that is refused rather than started. A shorter one is left to
# ZeroGPU, which admits it when the caller's own allowance covers it.
MAX_GPU_SECONDS = 900
SECONDS_PER_INPUT_SECOND = {"small": 0.1, "medium": 0.15, "large": 0.3}
def gpu_seconds(input_audio_path, variant, instruments_text, use_sampling, temperature):
return 20 + SECONDS_PER_INPUT_SECOND[variant] * sf.info(input_audio_path).duration
def process_fn(
input_audio_path: str,
variant: str,
instruments_text: str,
use_sampling: bool,
temperature: float,
) -> tuple[str, str]:
"""Transcribe the input audio to MIDI, plus a note on the Instruments field."""
# Checked before the GPU is reserved, so a request the model cannot take spends none of the
# caller's ZeroGPU allowance
if variant not in _WEIGHT_REVISIONS:
raise gr.Error(f"Unknown model size: {variant}")
if not input_audio_path:
raise gr.Error("Provide the music to transcribe.")
try:
duration = sf.info(input_audio_path).duration
except Exception as exc:
raise gr.Error(f"Could not read the audio file: {exc}") from exc
if duration == 0:
raise gr.Error("The audio file is empty.")
if gpu_seconds(input_audio_path, variant, instruments_text, use_sampling,
temperature) > MAX_GPU_SECONDS:
raise gr.Error(
f"The music is {duration:.0f} s long, more than one request can transcribe in the "
f"{MAX_GPU_SECONDS // 60} minutes it may run. Trim it, or use a smaller model."
)
return transcribe(input_audio_path, variant, instruments_text, use_sampling, temperature)
@spaces.GPU(duration=gpu_seconds)
@torch.inference_mode()
def transcribe(input_audio_path, variant, instruments_text, use_sampling, temperature):
model = _get_model(variant)
instruments, instrument_note = _resolve_instruments(instruments_text)
midi_bytes = model.transcribe_to_midi(
input_audio_path,
use_sampling=use_sampling,
temperature=temperature,
instruments=instruments,
)
with tempfile.NamedTemporaryFile(suffix=".mid", delete=False) as f:
f.write(midi_bytes)
output_midi_path = f.name
notes_text = f"{Path(input_audio_path).name}\n\nInstrument Matching\n{instrument_note}\n"
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
f.write(notes_text)
output_notes_path = f.name
return output_midi_path, output_notes_path
if __name__ == "__main__":
with gr.Blocks() as demo:
input_components = [
gr.Audio(type="filepath", label="Input Audio").harp_required(True).set_info((
"Music to transcribe. Trained on a wide range of material, from "
"classical to metal, and it names the instruments it hears."
)),
gr.Dropdown(
choices=["small", "medium", "large"],
value=DEFAULT_SIZE,
label="Model Size",
info="small = fastest, least accurate. medium = balanced (default, per repo). large = most accurate, slower.",
),
gr.Textbox(
value="",
label="Instruments",
info=f"Separate names with commas. Leave blank to let the model detect instruments on its own. Valid names: {_VALID_INSTRUMENTS}.",
),
gr.Checkbox(
label="Use Sampling",
info="Temperature sampling instead of greedy decoding (default: False, per repo).",
),
gr.Slider(
minimum=0.1,
maximum=2.0,
step=0.1,
value=1.0,
label="Temperature",
info="Only used when Use Sampling is on. Higher values = note choices are more varied and unpredictable. Lower values = values are closer to model's most confident guesses. (default: 1.0, per repo).",
),
]
output_components = [
gr.File(type="filepath", file_types=[".mid", ".midi"], label="Transcription").set_info(
"Transcribed MIDI notes."
),
gr.File(type="filepath", file_types=[".txt"], label="Instrument Matching").set_info(
"Which requested instrument names were matched or ignored."
),
]
build_endpoint(
model_card=model_card,
input_components=input_components,
output_components=output_components,
process_fn=process_fn,
)
demo.queue().launch(show_error=True, pwa=True, ssr_mode=False)