Clasificando 14,500 tracks con IA: Music Flamingo, pgvector y AMD ROCm

# El modelo acepta audio + texto y genera análisis
response = model.generate(
    audio=audio_waveform,
    prompt="Identify the genre, subgenre, and musical era of this track. "
           "Return only valid JSON: {\"genre\": \"\", \"subgenre\": \"\", \"era\": \"\"}",
)

HSA_STATUS_ERROR_OUT_OF_RESOURCES

environment:
  - HSA_OVERRIDE_GFX_VERSION=12.0.1
  - QUANT=int8
devices:
  - /dev/kfd
  - /dev/dri/renderD128

flowchart TD
    FS["Sistema de archivos\n/mnt/user/Music\n14,500 tracks"]
    SC["scanner.py\nregistra tracks en DB"]
    PG[("PostgreSQL 18 + pgvector\ntracks · track_analysis\nlyrics · track_embeddings")]
    WK["worker.py\ncola de jobs con\nFOR UPDATE SKIP LOCKED"]

    subgraph GPU["GPU — RX 9070 XT (221W cap)"]
        MF["Music Flamingo\nFlask API :8181\nINT8 · ROCm 7.2.3"]
    end

    subgraph CLOUD["Cloud"]
        OAI["OpenAI\ntext-embedding-3-small\n1536 dims"]
    end

    subgraph OLLAMA["Ollama — CPU/GPU"]
        GEM["gemma4:31b-cloud\nletras: traducción\ny análisis semántico"]
    end

    WEB["DuckDuckGo + Genius\nAZLyrics scraping"]

    FS --> SC --> PG
    PG --> WK
    WK -->|"music_analysis"| MF
    WK -->|"lyrics_fetch"| WEB
    WK -->|"lyrics_translate\nlyrics_process"| GEM
    WK -->|"embeddings_*"| OAI
    MF --> PG
    WEB --> PG
    GEM --> PG
    OAI --> PG

music_analysis
    ├── embeddings_audio
    ├── embeddings_description
    └── lyrics_fetch
            └── lyrics_translate
                    └── lyrics_process
                            └── embeddings_lyrics
                                    └── embeddings_combine

-- Análisis musical completo
SELECT genre, subgenre, era, genre_tags,
       mood, energy, atmosphere, intensity,
       bpm, musical_key, mode, instruments,
       production_style, mix_notes, description
FROM track_analysis WHERE track_id = 42;

-- Tres vectores de 1536 dims para similaridad semántica
SELECT audio_embedding, description_embedding, combined_embedding
FROM track_embeddings WHERE track_id = 42;

-- Tracks registrados por el scanner
CREATE TABLE tracks (
    id SERIAL PRIMARY KEY,
    file_path TEXT UNIQUE NOT NULL,
    artist TEXT, title TEXT, album TEXT,
    duration REAL, file_format TEXT,
    scanned_at TIMESTAMPTZ DEFAULT now()
);

-- Análisis del modelo de audio
CREATE TABLE track_analysis (
    track_id INT PRIMARY KEY REFERENCES tracks(id),
    genre TEXT, subgenre TEXT, era TEXT, genre_tags TEXT[],
    mood TEXT[], energy INT, atmosphere TEXT,
    emotional_arc TEXT, intensity TEXT,
    bpm REAL, bpm_confidence TEXT, musical_key TEXT, mode TEXT,
    instruments TEXT[], production_style TEXT,
    mix_notes TEXT, tempo_feel TEXT,
    description TEXT, description_embedding vector(1536),
    raw_responses JSONB, analyzed_at TIMESTAMPTZ
);

-- Letras y análisis semántico
CREATE TABLE lyrics (
    track_id INT PRIMARY KEY REFERENCES tracks(id),
    lyrics_original TEXT, lyrics_english TEXT, language TEXT,
    keywords TEXT[], topics TEXT[], themes TEXT[], sentiment TEXT,
    lyrics_embedding vector(1536),
    keywords_embedding vector(1536),
    topics_embedding vector(1536)
);

-- Embeddings consolidados
CREATE TABLE track_embeddings (
    track_id INT PRIMARY KEY REFERENCES tracks(id),
    audio_embedding    vector(1536),
    description_embedding vector(1536),
    combined_embedding vector(1536),
    model_used TEXT
);

# Límite mínimo que permite el driver: 221W
echo 221000000 > /sys/class/drm/card0/device/hwmon/hwmon1/power1_cap

sleep 10  # esperar a que cargue el driver amdgpu
HWMON=$(grep -rl '^amdgpu$' /sys/class/hwmon/*/name 2>/dev/null | head -1 | xargs dirname)
if [ -n "$HWMON" ]; then
    echo 221000000 > "$HWMON/power1_cap"
fi

def get_embedding(text: str) -> list[float]:
    resp = requests.post(
        "https://api.openai.com/v1/embeddings",
        headers={"Authorization": f"Bearer {config.OPENAI_API_KEY}"},
        json={"model": "text-embedding-3-small", "input": text, "dimensions": 1536},
        timeout=60,
    )
    resp.raise_for_status()
    return resp.json()["data"][0]["embedding"]

-- "Dame los 10 tracks más similares a esta canción"
SELECT t.artist, t.title,
       1 - (te.combined_embedding <=> $1::vector) AS similarity
FROM track_embeddings te
JOIN tracks t ON te.track_id = t.id
ORDER BY te.combined_embedding <=> $1::vector
LIMIT 10;

-- "Tracks melancólicos, bajo BPM, en menor, energía < 5"
SELECT t.artist, t.title, a.bpm, a.energy, a.mood
FROM track_analysis a
JOIN tracks t ON a.track_id = t.id
WHERE a.mode = 'minor'
  AND a.energy < 5
  AND a.bpm < 90
  AND 'melancholic' = ANY(a.mood)
ORDER BY a.energy ASC;