diff --git a/main/Analyser.cpp b/main/Analyser.cpp index 957bfe63..8d6789ab 100644 --- a/main/Analyser.cpp +++ b/main/Analyser.cpp @@ -14,6 +14,7 @@ */ #include "Analyser.h" +#include "RecordingPreview.h" #include "transform/TransformFactory.h" #include "transform/ModelTransformer.h" @@ -47,10 +48,14 @@ Analyser::Analyser() : m_document(0), m_paneStack(0), m_pane(0), + m_recordingPreview(new RecordingPreview(this)), m_currentCandidate(-1), m_candidatesVisible(false), m_currentAsyncHandle(0) { + connect(m_recordingPreview, SIGNAL(previewUpdated()), + this, SIGNAL(layersChanged())); + QSettings settings; settings.beginGroup("LayerDefaults"); settings.setValue @@ -77,7 +82,8 @@ Analyser::getAnalysisSettings() return { { "precision-analysis", false }, { "lowamp-analysis", true }, { "onset-analysis", true }, - { "prune-analysis", true } + { "prune-analysis", true }, + { "record-preview", false } }; } @@ -126,6 +132,53 @@ Analyser::analyseExistingFile() return doAllAnalyses(true); } +QString +Analyser::beginRecordingPreview() +{ + if (!m_document) return "Internal error: Analyser::beginRecordingPreview() called with no document present"; + + if (!m_pane) return "Internal error: Analyser::beginRecordingPreview() called with no pane present"; + + if (m_fileModel.isNone()) return "Internal error: Analyser::beginRecordingPreview() called with no model present"; + + // The preview draws into the pitch track, so we need one. If + // auto-analysis is switched off there won't be one yet, but asking + // to see analysis while recording is a clear enough request for it. + if (!m_layers[PitchTrack]) { + QString error = addAnalyses(); + if (error != "") return error; + } + + TimeValueLayer *pitchLayer = + qobject_cast(m_layers[PitchTrack]); + if (!pitchLayer) { + return "Internal error: Analyser::beginRecordingPreview() has no pitch track layer"; + } + + FlexiNoteLayer *noteLayer = + qobject_cast(m_layers[Notes]); + + return m_recordingPreview->begin(m_fileModel, pitchLayer, noteLayer); +} + +void +Analyser::recordingPreviewReachedFrame(sv_frame_t frame) +{ + m_recordingPreview->recordedTo(frame); +} + +bool +Analyser::isRecordingPreviewActive() const +{ + return m_recordingPreview->isActive(); +} + +void +Analyser::endRecordingPreview() +{ + m_recordingPreview->end(); +} + QString Analyser::doAllAnalyses(bool withPitchTrack) { @@ -170,6 +223,7 @@ void Analyser::fileClosed() { cerr << "Analyser::fileClosed" << endl; + m_recordingPreview->abandon(); m_layers.clear(); m_reAnalysisCandidates.clear(); m_currentCandidate = -1; diff --git a/main/Analyser.h b/main/Analyser.h index 918d239c..294c7172 100644 --- a/main/Analyser.h +++ b/main/Analyser.h @@ -36,6 +36,8 @@ class TimeValueLayer; class Layer; } +class RecordingPreview; + class Analyser : public QObject, public sv::Document::LayerCreationHandler { @@ -56,6 +58,31 @@ class Analyser : public QObject, // layers; return "" on success or error string on failure QString analyseExistingFile(); + /** + * Start filling in the pitch track as a recording is made, so that + * the performer can see something as they go. Everything this adds + * is removed again by endRecordingPreview(), because the pitch and + * note layers are regenerated in full when recording stops. Returns + * "" on success or an error string on failure. + */ + QString beginRecordingPreview(); + + /** + * Note that the recording being previewed has reached the given + * frame. + */ + void recordingPreviewReachedFrame(sv::sv_frame_t frame); + + /** + * Return true if a recording preview is currently running. + */ + bool isRecordingPreviewActive() const; + + /** + * Stop previewing and remove everything the preview added. + */ + void endRecordingPreview(); + // Discard any layers etc associated with the current document void fileClosed(); @@ -245,6 +272,8 @@ protected slots: mutable std::map m_layers; + RecordingPreview *m_recordingPreview; + sv::Clipboard m_preAnalysis; sv::Selection m_reAnalysingSelection; FrequencyRange m_reAnalysingRange; diff --git a/main/MainWindow.cpp b/main/MainWindow.cpp index cd9fa4af..28867685 100644 --- a/main/MainWindow.cpp +++ b/main/MainWindow.cpp @@ -122,7 +122,8 @@ MainWindow::MainWindow(AudioMode audioMode, m_keyReference(new KeyReference()), m_selectionAnchor(0), m_withSonification(withSonification), - m_withSpectrogram(withSpectrogram) + m_withSpectrogram(withSpectrogram), + m_recordPreviewAttempted(false) { setWindowTitle(QApplication::applicationName()); @@ -774,6 +775,12 @@ MainWindow::setupAnalysisMenu() connect(m_autoAnalyse, SIGNAL(triggered()), this, SLOT(autoAnalysisToggled())); menu->addAction(m_autoAnalyse); + m_recordPreview = new QAction(tr("Show Pitch Track While &Recording"), this); + m_recordPreview->setStatusTip(tr("Fill in the pitch track as a recording is made. The analysis is redone in full when recording stops.")); + m_recordPreview->setCheckable(true); + connect(m_recordPreview, SIGNAL(triggered()), this, SLOT(recordPreviewToggled())); + menu->addAction(m_recordPreview); + action = new QAction(tr("&Analyse Now!"), this); action->setStatusTip(tr("Trigger analysis of pitches and notes. (This will delete all existing pitches and notes.)")); connect(action, SIGNAL(triggered()), this, SLOT(analyseNow())); @@ -846,7 +853,8 @@ MainWindow::updateAnalyseStates() { "precision-analysis", m_precise }, { "lowamp-analysis", m_lowamp }, { "onset-analysis", m_onset }, - { "prune-analysis", m_prune } + { "prune-analysis", m_prune }, + { "record-preview", m_recordPreview } }; auto keyMap = Analyser::getAnalysisSettings(); @@ -881,6 +889,69 @@ MainWindow::autoAnalysisToggled() updateAnalyseStates(); } +void +MainWindow::recordPreviewToggled() +{ + QAction *a = qobject_cast(sender()); + if (!a) return; + + QSettings settings; + settings.beginGroup("Analyser"); + settings.setValue("record-preview", a->isChecked()); + settings.endGroup(); + + updateAnalyseStates(); +} + +void +MainWindow::recordStatusChanged(bool recording) +{ + if (recording) { + m_recordPreviewAttempted = false; + return; + } + + // Everything the preview added is removed here. The pitch and note + // layers are then regenerated in full from the completed recording, + // exactly as they are without this feature. + m_analyser->endRecordingPreview(); +} + +void +MainWindow::recordDurationChanged(sv_frame_t frame, sv_samplerate_t rate) +{ + MainWindowBase::recordDurationChanged(frame, rate); + + if (!m_analyser->isRecordingPreviewActive()) { + + if (m_recordPreviewAttempted) return; + m_recordPreviewAttempted = true; + + // Not started on recordStatusChanged(true): that is emitted from + // startRecording(), before record() has closed the old session, + // created the new document, set the main model and built the + // analysis layers. Starting there would attach the preview to + // the session that is about to be discarded. By the time the + // first duration update arrives all of that has happened. + + QSettings settings; + settings.beginGroup("Analyser"); + bool preview = settings.value("record-preview", false).toBool(); + settings.endGroup(); + + if (!preview) return; + + QString error = m_analyser->beginRecordingPreview(); + if (error != "") { + SVCERR << "MainWindow::recordDurationChanged: unable to preview: " + << error << endl; + return; + } + } + + m_analyser->recordingPreviewReachedFrame(frame); +} + void MainWindow::precisionAnalysisToggled() { @@ -1078,6 +1149,8 @@ MainWindow::setupToolbars() connect(recordAction, SIGNAL(triggered()), this, SLOT(record())); connect(m_recordTarget, SIGNAL(recordStatusChanged(bool)), recordAction, SLOT(setChecked(bool))); + connect(m_recordTarget, SIGNAL(recordStatusChanged(bool)), + this, SLOT(recordStatusChanged(bool))); connect(m_recordTarget, SIGNAL(recordCompleted()), this, SLOT(analyseNow())); connect(this, SIGNAL(canRecord(bool)), diff --git a/main/MainWindow.h b/main/MainWindow.h index f14d5ca9..1f6bb65e 100644 --- a/main/MainWindow.h +++ b/main/MainWindow.h @@ -89,6 +89,8 @@ protected slots: virtual void analyseNow(); virtual void resetAnalyseOptions(); virtual void autoAnalysisToggled(); + virtual void recordPreviewToggled(); + virtual void recordStatusChanged(bool); virtual void precisionAnalysisToggled(); virtual void lowampAnalysisToggled(); virtual void onsetAnalysisToggled(); @@ -205,6 +207,7 @@ protected slots: bool m_intelligentActionOn; // GF: !!! temporary QAction *m_autoAnalyse; + QAction *m_recordPreview; QAction *m_precise; QAction *m_lowamp; QAction *m_onset; @@ -231,6 +234,10 @@ protected slots: bool m_withSonification; bool m_withSpectrogram; + // Preview is started on the first duration update rather than when + // recording starts; this stops us retrying every update if it fails + bool m_recordPreviewAttempted; + Analyser::FrequencyRange m_pendingConstraint; QString exportToSVL(QString path, sv::Layer *layer); @@ -254,6 +261,8 @@ protected slots: bool checkSaveModified(); bool waitForInitialAnalysis(); + virtual void recordDurationChanged(sv::sv_frame_t, sv::sv_samplerate_t); + virtual void updateVisibleRangeDisplay(sv::Pane *p) const; virtual void updatePositionStatusDisplays() const; diff --git a/main/PreviewChunk.cpp b/main/PreviewChunk.cpp new file mode 100644 index 00000000..c09ca827 --- /dev/null +++ b/main/PreviewChunk.cpp @@ -0,0 +1,66 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2012 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "PreviewChunk.h" + +using namespace sv; + +namespace PreviewChunk { + +std::optional +nextRange(sv_frame_t analysedTo, + sv_frame_t recordedTo, + sv_frame_t minFrames, + sv_frame_t maxFrames, + sv_frame_t revisitFrames) +{ + if (analysedTo < 0) analysedTo = 0; + + if (recordedTo <= analysedTo) { + return {}; + } + + if (recordedTo - analysedTo < minFrames) { + return {}; + } + + sv_frame_t to = recordedTo; + + if (maxFrames > 0 && to - analysedTo > maxFrames) { + to = analysedTo + maxFrames; + } + + sv_frame_t from = analysedTo - revisitFrames; + if (from < 0) from = 0; + + return Range { from, to }; +} + +EventVector +withinRange(const EventVector &events, const Range &range) +{ + EventVector result; + result.reserve(events.size()); + + for (const auto &e: events) { + if (e.getFrame() >= range.from && e.getFrame() < range.to) { + result.push_back(e); + } + } + + return result; +} + +} diff --git a/main/PreviewChunk.h b/main/PreviewChunk.h new file mode 100644 index 00000000..376e2b34 --- /dev/null +++ b/main/PreviewChunk.h @@ -0,0 +1,90 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2012 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef TONY_PREVIEW_CHUNK_H +#define TONY_PREVIEW_CHUNK_H + +#include "base/BaseTypes.h" +#include "base/Event.h" + +#include + +/** + * Pure helpers for previewing analysis of a recording that is still + * being made. These have no dependency on the document, models or UI, + * so they can be tested directly. + */ +namespace PreviewChunk { + +struct Range { + sv::sv_frame_t from; + sv::sv_frame_t to; + + sv::sv_frame_t length() const { return to - from; } + + bool operator==(const Range &r) const { + return from == r.from && to == r.to; + } +}; + +/** + * Return the next region of a growing recording to analyse, given how + * far analysis has already reached and how far the recording has got. + * + * The region starts revisitFrames before the end of the previous one, + * so each pass re-analyses a short tail of what it did last time. The + * caller is expected to discard its earlier results for that tail and + * replace them wholesale. + * + * That matters for notes, which have duration and would otherwise be + * chopped in two wherever a region boundary fell in the middle of one. + * Revisiting gives the plugin's note tracker the surrounding audio and + * lets it produce the note whole. Because a preview owns everything it + * draws, replacing a region is just a delete and an add -- there is + * nothing to merge, and no heuristic deciding which of two overlapping + * notes to believe. + * + * The region never starts later than the previous one ended, so a + * duration notification arriving out of order cannot make the preview + * rewrite material it has already settled. + * + * Returns nothing if there is less than minFrames of new audio, so we + * don't spend more time starting transforms than running them. The new + * audio is clamped to maxFrames (when positive) so that a long stall + * results in several ordinary passes rather than one huge one. + */ +std::optional nextRange(sv::sv_frame_t analysedTo, + sv::sv_frame_t recordedTo, + sv::sv_frame_t minFrames, + sv::sv_frame_t maxFrames, + sv::sv_frame_t revisitFrames); + +/** + * Return only those events whose frame lies within the given range. + * + * A transform output is expected to be expressed in absolute frames of + * the source audio, but that depends on the plugin: a Vamp plugin that + * derives its timestamps from a frame counter rather than from the + * timestamps the host supplies will produce output relative to the + * start of the region instead. Discarding anything outside the region + * we asked for means such an output shows up as a preview that stays + * empty, rather than as points scattered across the recording. + */ +sv::EventVector withinRange(const sv::EventVector &events, + const Range &range); + +} + +#endif diff --git a/main/RecordingPreview.cpp b/main/RecordingPreview.cpp new file mode 100644 index 00000000..92c405e3 --- /dev/null +++ b/main/RecordingPreview.cpp @@ -0,0 +1,380 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2012 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "RecordingPreview.h" + +#include "base/Debug.h" +#include "base/RealTime.h" +#include "transform/Transform.h" +#include "transform/TransformFactory.h" +#include "transform/ModelTransformerFactory.h" +#include "data/model/WaveFileModel.h" +#include "data/model/SparseTimeValueModel.h" +#include "data/model/NoteModel.h" +#include "layer/TimeValueLayer.h" +#include "layer/FlexiNoteLayer.h" + +#include + +using namespace sv; + +RecordingPreview::RecordingPreview(QObject *parent) : + QObject(parent), + m_active(false), + m_sampleRate(0), + m_currentRange({ 0, 0 }), + m_analysedTo(0), + m_recordedTo(0) +{ +} + +RecordingPreview::~RecordingPreview() +{ + releaseTransforms(); +} + +sv_frame_t +RecordingPreview::toFrames(double seconds) const +{ + return sv_frame_t(seconds * m_sampleRate); +} + +QString +RecordingPreview::begin(ModelId sourceModel, + TimeValueLayer *targetPitchLayer, + FlexiNoteLayer *targetNoteLayer) +{ + abandon(); + + if (!targetPitchLayer && !targetNoteLayer) { + return "Internal error: RecordingPreview::begin() called with no target layers"; + } + + auto source = ModelById::getAs(sourceModel); + if (!source) { + return "Internal error: RecordingPreview::begin() called with no source model"; + } + + TransformFactory *tf = TransformFactory::getInstance(); + + QString f0Id = QString("%1%2").arg(PYIN_TRANSFORM_BASE).arg(PYIN_F0_OUTPUT); + QString noteId = QString("%1%2").arg(PYIN_TRANSFORM_BASE).arg(PYIN_NOTE_OUTPUT); + + if (!tf->haveTransform(f0Id) || !tf->haveTransform(noteId)) { + return tr("Transform \"%1\" not found. Unable to preview analysis while " + "recording.

Is the pYIN Vamp plugin correctly installed?") + .arg(f0Id); + } + + m_sourceModel = sourceModel; + m_sampleRate = source->getSampleRate(); + + m_targetPitchLayer = targetPitchLayer; + m_targetPitchModel = targetPitchLayer ? targetPitchLayer->getModel() : ModelId(); + + m_targetNoteLayer = targetNoteLayer; + m_targetNoteModel = targetNoteLayer ? targetNoteLayer->getModel() : ModelId(); + + m_analysedTo = 0; + m_recordedTo = 0; + m_addedPitch.clear(); + m_addedNotes.clear(); + m_active = true; + + SVDEBUG << "RecordingPreview::begin: previewing into pitch model " + << m_targetPitchModel << " and note model " << m_targetNoteModel + << endl; + + return ""; +} + +void +RecordingPreview::recordedTo(sv_frame_t frame) +{ + if (!m_active) return; + + if (frame > m_recordedTo) { + m_recordedTo = frame; + } + + if (!m_pitchOutput.isNone() || !m_noteOutput.isNone()) { + // A pass is already running; it will pick up the new mark when + // it finishes + return; + } + + auto range = PreviewChunk::nextRange(m_analysedTo, m_recordedTo, + toFrames(MIN_CHUNK_SECONDS), + toFrames(MAX_CHUNK_SECONDS), + toFrames(REVISIT_SECONDS)); + if (range) { + startPass(*range); + } +} + +void +RecordingPreview::startPass(const PreviewChunk::Range &range) +{ + auto source = ModelById::getAs(m_sourceModel); + if (!source) { + abandon(); + return; + } + + TransformFactory *tf = TransformFactory::getInstance(); + + QString f0Id = QString("%1%2").arg(PYIN_TRANSFORM_BASE).arg(PYIN_F0_OUTPUT); + + Transform transform = tf->getDefaultTransformFor(f0Id, m_sampleRate); + transform.setStepSize(PYIN_STEP_SIZE); + transform.setBlockSize(PYIN_BLOCK_SIZE); + transform.setStartTime(RealTime::frame2RealTime(range.from, m_sampleRate)); + transform.setDuration(RealTime::frame2RealTime(range.length(), m_sampleRate)); + + // Both outputs from a single run of the plugin: transformMultiple + // requires transforms differing only in output identifier + Transforms transforms; + transforms.push_back(transform); + transform.setOutput(PYIN_NOTE_OUTPUT); + transforms.push_back(transform); + + QString message; + + // Not Document::createDerivedLayers: we want the output models only, + // with no layers, no registration with the document and nothing + // added to the undo history. The models returned here belong to us. + std::vector outputs = ModelTransformerFactory::getInstance()-> + transformMultiple(transforms, ModelTransformer::Input(m_sourceModel), + message); + + if (outputs.size() < 2) { + SVDEBUG << "RecordingPreview::startPass: transform failed: " + << message << endl; + for (ModelId id: outputs) { + ModelTransformerFactory::getInstance()->cancel(id); + ModelById::release(id); + } + // Move past this region rather than retrying it forever + m_analysedTo = range.to; + return; + } + + m_pitchOutput = outputs[0]; + m_noteOutput = outputs[1]; + m_currentRange = range; + + bool complete = true; + + for (ModelId id: { m_pitchOutput, m_noteOutput }) { + auto model = ModelById::get(id); + if (!model) { + complete = false; + continue; + } + // Queued: completionChanged is emitted from the transform's own + // thread, and the handler releases the model that emitted it. + // Going through the event loop keeps us from doing that while + // the signal is still being delivered. + connect(model.get(), SIGNAL(completionChanged(ModelId)), + this, SLOT(transformCompletionChanged(ModelId)), + Qt::QueuedConnection); + if (model->getCompletion() != 100) complete = false; + } + + // The transforms may have finished already, in which case the + // signals have been and gone and nothing further would arrive + if (complete) { + collectPass(); + } +} + +void +RecordingPreview::transformCompletionChanged(ModelId) +{ + if (m_pitchOutput.isNone() || m_noteOutput.isNone()) return; + + for (ModelId id: { m_pitchOutput, m_noteOutput }) { + auto model = ModelById::get(id); + if (!model || model->getCompletion() != 100) return; + } + + collectPass(); +} + +void +RecordingPreview::collectPass() +{ + const PreviewChunk::Range range = m_currentRange; + + auto pitchOut = ModelById::getAs(m_pitchOutput); + auto noteOut = ModelById::getAs(m_noteOutput); + + EventVector pitchEvents, noteEvents; + + if (pitchOut) { + // pYIN's smoothedpitchtrack is a fixed-sample-rate output whose + // features carry the host's block timestamps, so these frames + // are already absolute. + pitchEvents = PreviewChunk::withinRange(pitchOut->getAllEvents(), range); + } + + if (noteOut) { + // The notes output is different: it is variable-sample-rate and + // derives its timestamps from a frame index counting from zero, + // ignoring the host's block timestamps, so these frames are + // relative to the start of the analysed region. + EventVector relative = noteOut->getAllEvents(); + noteEvents.reserve(relative.size()); + for (const Event &e: relative) { + noteEvents.push_back(e.withFrame(e.getFrame() + range.from)); + } + noteEvents = PreviewChunk::withinRange(noteEvents, range); + } + + releaseTransforms(); + + // Replace, rather than merge: drop whatever we put in this region + // last time before adding what we have now + removeAddedFrom(range.from); + + auto pitchTarget = ModelById::getAs(m_targetPitchModel); + if (pitchTarget && m_targetPitchLayer && + m_targetPitchLayer->getModel() == m_targetPitchModel) { + for (const Event &e: pitchEvents) { + pitchTarget->add(e); + m_addedPitch.push_back(e); + } + } + + auto noteTarget = ModelById::getAs(m_targetNoteModel); + if (noteTarget && m_targetNoteLayer && + m_targetNoteLayer->getModel() == m_targetNoteModel) { + for (const Event &e: noteEvents) { + noteTarget->add(e); + m_addedNotes.push_back(e); + } + } + + m_analysedTo = range.to; + + if (!pitchEvents.empty() || !noteEvents.empty()) { + emit previewUpdated(); + } + + // Pick up anything that arrived while this pass was running + auto next = PreviewChunk::nextRange(m_analysedTo, m_recordedTo, + toFrames(MIN_CHUNK_SECONDS), + toFrames(MAX_CHUNK_SECONDS), + toFrames(REVISIT_SECONDS)); + if (next) { + startPass(*next); + } +} + +void +RecordingPreview::releaseTransforms() +{ + for (ModelId *id: { &m_pitchOutput, &m_noteOutput }) { + + if (id->isNone()) continue; + + ModelId output = *id; + *id = {}; + + auto model = ModelById::get(output); + if (model) { + disconnect(model.get(), SIGNAL(completionChanged(ModelId)), + this, SLOT(transformCompletionChanged(ModelId))); + } + + // Unconditionally, including when the transform has already + // reported completion: reaching 100 happens inside run(), so the + // thread may still be tearing down. cancel() waits for it to + // exit, which is what makes it safe to release the model here. + ModelTransformerFactory::getInstance()->cancel(output); + ModelById::release(output); + } +} + +void +RecordingPreview::removeAddedFrom(sv_frame_t frame) +{ + auto pitchTarget = ModelById::getAs(m_targetPitchModel); + bool pitchUsable = (pitchTarget && m_targetPitchLayer && + m_targetPitchLayer->getModel() == m_targetPitchModel); + + auto noteTarget = ModelById::getAs(m_targetNoteModel); + bool noteUsable = (noteTarget && m_targetNoteLayer && + m_targetNoteLayer->getModel() == m_targetNoteModel); + + // Each pass prunes from its region start and then appends that + // region's events in order, and the region start only ever moves + // forward, so these stay sorted by frame. That lets us find the + // cut point rather than scanning: this runs on every pass, and the + // vectors reach six figures over a long take. + auto prune = [frame](EventVector &added, bool usable, auto target) { + auto split = std::lower_bound + (added.begin(), added.end(), frame, + [](const Event &e, sv_frame_t f) { return e.getFrame() < f; }); + if (usable) { + for (auto i = split; i != added.end(); ++i) { + target->remove(*i); + } + } + added.erase(split, added.end()); + }; + + prune(m_addedPitch, pitchUsable, pitchTarget); + prune(m_addedNotes, noteUsable, noteTarget); +} + +void +RecordingPreview::end() +{ + if (!m_active) return; + + SVDEBUG << "RecordingPreview::end: removing " << m_addedPitch.size() + << " preview pitch point(s) and " << m_addedNotes.size() + << " preview note(s)" << endl; + + releaseTransforms(); + + // Everything, from frame zero + removeAddedFrom(0); + + m_active = false; + m_targetPitchLayer = nullptr; + m_targetNoteLayer = nullptr; + m_targetPitchModel = {}; + m_targetNoteModel = {}; + m_sourceModel = {}; +} + +void +RecordingPreview::abandon() +{ + releaseTransforms(); + + m_addedPitch.clear(); + m_addedNotes.clear(); + m_active = false; + m_targetPitchLayer = nullptr; + m_targetNoteLayer = nullptr; + m_targetPitchModel = {}; + m_targetNoteModel = {}; + m_sourceModel = {}; + m_analysedTo = 0; + m_recordedTo = 0; +} diff --git a/main/RecordingPreview.h b/main/RecordingPreview.h new file mode 100644 index 00000000..70ee32a3 --- /dev/null +++ b/main/RecordingPreview.h @@ -0,0 +1,156 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2012 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef TONY_RECORDING_PREVIEW_H +#define TONY_RECORDING_PREVIEW_H + +#include "PreviewChunk.h" + +#include "base/BaseTypes.h" +#include "data/model/Model.h" + +#include +#include + +namespace sv { +class TimeValueLayer; +class FlexiNoteLayer; +} + +/** + * Fills in the pitch track and notes as a recording is made, so that the + * performer can see something while singing or playing. + * + * This is only ever a preview. When the recording stops, the pitch and + * note layers are regenerated in full from the completed audio (by + * Document::replaceModel, via MainWindowBase's refreshModel call), so + * nothing produced here survives into the saved session. Accordingly + * this class removes everything it added when the recording finishes, + * leaving the layers exactly as they would have been without it. + * + * Each pass runs a pYIN transform over one region of the recording so + * far, and the regions overlap: each pass revisits a short tail of the + * previous one and replaces the preview's contents there. Notes have + * duration and would otherwise be cut in two wherever a region boundary + * fell inside one; revisiting lets the note tracker see the surrounding + * audio and emit the note whole. Because everything drawn here belongs + * to the preview, replacing a region is a delete followed by an add, + * with nothing to merge and no heuristic choosing between overlapping + * notes. + * + * Only one transform runs at a time. Duration updates arriving while one + * is running raise the mark for the next pass rather than starting + * another. + * + * The output models come from ModelTransformerFactory rather than from + * Document, so they belong to us: no layer is created for them, they are + * not registered with the document, and nothing is added to the undo + * history. + */ +class RecordingPreview : public QObject +{ + Q_OBJECT + +public: + explicit RecordingPreview(QObject *parent = nullptr); + virtual ~RecordingPreview(); + + /** + * Begin previewing into the given layers, analysing the given source + * (recording) model. Either target layer may be null, in which case + * that part of the preview is skipped. Returns "" on success or an + * error string on failure. + */ + QString begin(sv::ModelId sourceModel, + sv::TimeValueLayer *targetPitchLayer, + sv::FlexiNoteLayer *targetNoteLayer); + + /** + * Note that the recording has reached the given frame. Starts a pass + * if one is not already running and there is enough new audio. + */ + void recordedTo(sv::sv_frame_t frame); + + /** + * The recording has finished: cancel any running analysis and remove + * everything this preview added. + */ + void end(); + + /** + * Abandon the preview without touching the targets, for use when + * they are being replaced anyway. + */ + void abandon(); + + bool isActive() const { return m_active; } + +signals: + void previewUpdated(); + +protected slots: + void transformCompletionChanged(sv::ModelId); + +protected: + void startPass(const PreviewChunk::Range &range); + void collectPass(); + void releaseTransforms(); + void removeAddedFrom(sv::sv_frame_t frame); + + // Enough new audio to be worth starting a transform for, but short + // enough to feel responsive + static constexpr double MIN_CHUNK_SECONDS = 0.25; + + // Bound the work in a single pass, so a stall produces several + // ordinary passes rather than one very long one + static constexpr double MAX_CHUNK_SECONDS = 5.0; + + // How much of the previous pass to re-analyse and replace. Needs to + // comfortably exceed the length of a note for notes to come out + // whole across a boundary. + static constexpr double REVISIT_SECONDS = 1.5; + + static const int PYIN_STEP_SIZE = 256; + static const int PYIN_BLOCK_SIZE = 2048; + + static constexpr const char *PYIN_TRANSFORM_BASE = "vamp:pyin:pyin:"; + static constexpr const char *PYIN_F0_OUTPUT = "smoothedpitchtrack"; + static constexpr const char *PYIN_NOTE_OUTPUT = "notes"; + + sv::sv_frame_t toFrames(double seconds) const; + + bool m_active; + + sv::ModelId m_sourceModel; + sv::sv_samplerate_t m_sampleRate; + + QPointer m_targetPitchLayer; + sv::ModelId m_targetPitchModel; + sv::EventVector m_addedPitch; + + QPointer m_targetNoteLayer; + sv::ModelId m_targetNoteModel; + sv::EventVector m_addedNotes; + + sv::ModelId m_pitchOutput; + sv::ModelId m_noteOutput; + + PreviewChunk::Range m_currentRange; + + sv::sv_frame_t m_analysedTo; + sv::sv_frame_t m_recordedTo; +}; + +#endif diff --git a/main/test/TestPreviewChunk.h b/main/test/TestPreviewChunk.h new file mode 100644 index 00000000..3929f435 --- /dev/null +++ b/main/test/TestPreviewChunk.h @@ -0,0 +1,286 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef TEST_PREVIEW_CHUNK_H +#define TEST_PREVIEW_CHUNK_H + +#include "../PreviewChunk.h" + +#include +#include + +using namespace sv; + +class TestPreviewChunk : public QObject +{ + Q_OBJECT + +private: + static const sv_frame_t MIN = 11025; + static const sv_frame_t MAX = 220500; + + static Event pitch(sv_frame_t frame, float value) { + return Event(frame, value, QString()); + } + +private slots: + + // ---- nextRange -------------------------------------------------- + + void nothingRecordedYet() { + + QCOMPARE(PreviewChunk::nextRange(0, 0, MIN, MAX, 0).has_value(), false); + } + + void tooLittleNewAudio() { + + // Duration notifications arrive far more often than it is worth + // starting a transform + QCOMPARE(PreviewChunk::nextRange(0, MIN - 1, MIN, MAX, 0).has_value(), + false); + } + + void exactlyEnoughNewAudio() { + + auto r = PreviewChunk::nextRange(0, MIN, MIN, MAX, 0); + QVERIFY(r.has_value()); + QCOMPARE(r->from, sv_frame_t(0)); + QCOMPARE(r->to, MIN); + } + + void rangeStartsWhereTheLastOneEnded() { + + // Adjacent and non-overlapping, so results can simply be + // concatenated with nothing to merge + auto first = PreviewChunk::nextRange(0, 50000, MIN, MAX, 0); + QVERIFY(first.has_value()); + + auto second = PreviewChunk::nextRange(first->to, 100000, MIN, MAX, 0); + QVERIFY(second.has_value()); + + QCOMPARE(second->from, first->to); + QVERIFY(second->from >= first->to); + } + + void neverRunsBackwards() { + + // A duration notification that arrives out of order, or a + // recording that restarts, must not make the preview rewrite + // what it has already drawn + QCOMPARE(PreviewChunk::nextRange(100000, 50000, MIN, MAX, 0).has_value(), + false); + } + + void longStallIsBrokenIntoChunks() { + + // If analysis falls a long way behind we want several ordinary + // chunks rather than one enormous one + auto r = PreviewChunk::nextRange(0, MAX * 3, MIN, MAX, 0); + QVERIFY(r.has_value()); + QCOMPARE(r->from, sv_frame_t(0)); + QCOMPARE(r->to, MAX); + QCOMPARE(r->length(), MAX); + } + + void chunksEventuallyCatchUp() { + + // Repeatedly applying nextRange against a fixed end must + // terminate, having covered the whole span exactly once + const sv_frame_t recordedTo = MAX * 3 + 5000; + + sv_frame_t at = 0; + int iterations = 0; + + while (auto r = PreviewChunk::nextRange(at, recordedTo, MIN, MAX, 0)) { + QCOMPARE(r->from, at); + QVERIFY(r->to > r->from); + at = r->to; + QVERIFY(++iterations < 100); + } + + // Whatever is left is below the minimum chunk size + QVERIFY(recordedTo - at < MIN); + } + + void unboundedWhenMaxIsZero() { + + auto r = PreviewChunk::nextRange(0, 10 * MAX, MIN, 0, 0); + QVERIFY(r.has_value()); + QCOMPARE(r->to, 10 * MAX); + } + + void negativeStartIsClamped() { + + auto r = PreviewChunk::nextRange(-500, 50000, MIN, MAX, 0); + QVERIFY(r.has_value()); + QCOMPARE(r->from, sv_frame_t(0)); + } + + // ---- revisiting ------------------------------------------------- + + void revisitReachesBackIntoThepreviousRegion() { + + // Notes have duration and would be cut in two at a region + // boundary. Reaching back lets the note tracker see the + // surrounding audio and emit the note whole; the caller replaces + // its earlier results for the revisited part. + const sv_frame_t revisit = 66150; // 1.5s at 44.1kHz + + auto r = PreviewChunk::nextRange(200000, 250000, MIN, MAX, revisit); + QVERIFY(r.has_value()); + QCOMPARE(r->from, sv_frame_t(200000 - revisit)); + QCOMPARE(r->to, sv_frame_t(250000)); + } + + void revisitDoesNotChangeHowFarWeGet() { + + // Only the start moves back: the region must still end at the + // recording head, or analysis would never catch up + auto plain = PreviewChunk::nextRange(200000, 250000, MIN, MAX, 0); + auto revisited = PreviewChunk::nextRange(200000, 250000, MIN, MAX, 66150); + + QVERIFY(plain.has_value() && revisited.has_value()); + QCOMPARE(revisited->to, plain->to); + QVERIFY(revisited->from < plain->from); + } + + void revisitIsClampedAtTheStartOfTheRecording() { + + auto r = PreviewChunk::nextRange(20000, 50000, MIN, MAX, 66150); + QVERIFY(r.has_value()); + QCOMPARE(r->from, sv_frame_t(0)); + } + + void revisitStillTerminates() { + + // Revisiting must not stop analysis advancing: the end still + // moves forward every pass even though the start moves back + const sv_frame_t recordedTo = MAX * 3 + 5000; + const sv_frame_t revisit = 66150; + + sv_frame_t at = 0; + int iterations = 0; + + while (auto r = PreviewChunk::nextRange(at, recordedTo, MIN, MAX, + revisit)) { + QVERIFY(r->to > at); + at = r->to; + QVERIFY(++iterations < 100); + } + + QVERIFY(recordedTo - at < MIN); + } + + void addedEventsStaySortedAcrossPasses() { + + // RecordingPreview relies on this: it prunes its record of added + // events from the region start and then appends that region's + // events, so the record stays sorted by frame and can be pruned + // by binary search rather than by scanning. That matters because + // the record reaches six figures over a long recording and is + // pruned on every pass. + const sv_frame_t revisit = 66150; + const sv_frame_t step = 20000; + + sv_frame_t analysedTo = 0; + sv_frame_t recordedTo = 0; + sv_frame_t lastFrom = -1; + + for (int pass = 0; pass < 40; ++pass) { + + recordedTo += step; + + auto r = PreviewChunk::nextRange(analysedTo, recordedTo, + MIN, MAX, revisit); + if (!r) continue; + + // The region start must never go backwards, or a prune + // would leave later events in front of earlier ones + QVERIFY(r->from >= lastFrom); + lastFrom = r->from; + + // and must not exceed where we had reached, or a pass would + // leave a gap it never fills + QVERIFY(r->from <= analysedTo); + + analysedTo = r->to; + } + } + + // ---- withinRange ------------------------------------------------ + + void withinRangeKeepsEventsInside() { + + EventVector events { + pitch(1000, 440.f), + pitch(1500, 441.f), + pitch(1999, 442.f) + }; + + auto kept = PreviewChunk::withinRange(events, { 1000, 2000 }); + QCOMPARE(int(kept.size()), 3); + } + + void withinRangeIsHalfOpen() { + + EventVector events { + pitch(999, 440.f), // before + pitch(1000, 441.f), // first frame, kept + pitch(2000, 442.f) // one past the end, dropped + }; + + auto kept = PreviewChunk::withinRange(events, { 1000, 2000 }); + QCOMPARE(int(kept.size()), 1); + QCOMPARE(kept[0].getFrame(), sv_frame_t(1000)); + } + + void withinRangeRejectsRelativeFrames() { + + // A plugin whose output is relative to the start of the analysed + // region rather than absolute would land near frame zero. Those + // events must be discarded rather than drawn in the wrong place: + // an empty preview is a far better failure than a pitch track + // scattered across the recording. + EventVector relative { + pitch(0, 440.f), + pitch(256, 441.f), + pitch(512, 442.f) + }; + + auto kept = PreviewChunk::withinRange(relative, { 500000, 550000 }); + QCOMPARE(int(kept.size()), 0); + } + + void withinRangeRejectsDoubledFrames() { + + // The specific failure this guards against: frames offset by the + // region start a second time, landing at twice the true position + const sv_frame_t from = 500000, to = 550000; + + EventVector doubled { + pitch(from * 2, 440.f), + pitch(from * 2 + 256, 441.f) + }; + + QCOMPARE(int(PreviewChunk::withinRange(doubled, { from, to }).size()), 0); + } + + void withinRangeOfEmptyIsEmpty() { + + QCOMPARE(int(PreviewChunk::withinRange(EventVector(), + { 0, 1000 }).size()), 0); + } +}; + +#endif diff --git a/main/test/tony-main-test.cpp b/main/test/tony-main-test.cpp new file mode 100644 index 00000000..442aa305 --- /dev/null +++ b/main/test/tony-main-test.cpp @@ -0,0 +1,46 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "TestPreviewChunk.h" + +#include "system/Init.h" + +#include + +int main(int argc, char *argv[]) +{ + int good = 0, bad = 0; + + svSystemSpecificInitialisation(); + + QCoreApplication app(argc, argv); + app.setOrganizationName("sonic-visualiser"); + app.setApplicationName("test-tony-main"); + + { + TestPreviewChunk t; + if (QTest::qExec(&t, argc, argv) == 0) ++good; + else ++bad; + } + + (void)good; + + if (bad > 0) { + SVCERR << "\n********* " << bad << " test suite(s) failed!\n" << endl; + return 1; + } else { + SVCERR << "All tests passed" << endl; + return 0; + } +} diff --git a/meson.build b/meson.build index 41c9392d..c5e61ed7 100644 --- a/meson.build +++ b/meson.build @@ -191,6 +191,18 @@ elif system == 'darwin' general_link_args += [ '-mmacosx-version-min=10.15' ] + + # Qt's qyieldcpu.h (through 6.9.x at least) calls the ACLE intrinsic + # __yield() under __has_builtin(__yield) without including + # . Apple clang 21 reports the builtin as available but + # makes the resulting implicit declaration an error, so every + # translation unit that reaches fails to compile. + # Force-including the ACLE header supplies the declaration. + if architecture == 'aarch64' and meson.get_compiler('cpp').has_header('arm_acle.h') + general_defines += [ + '-include', 'arm_acle.h' + ] + endif feature_defines = [ '-DHAVE_BZ2', @@ -1023,12 +1035,20 @@ tony_main_files = [ 'main/Analyser.cpp', 'main/MainWindow.cpp', 'main/NetworkPermissionTester.cpp', + 'main/PreviewChunk.cpp', + 'main/RecordingPreview.cpp', ] tony_main_moc_files = qt.preprocess( moc_headers: [ 'main/MainWindow.h', 'main/Analyser.h', + 'main/RecordingPreview.h', +]) + +tony_main_test_moc_files = qt.preprocess( + moc_headers: [ + 'main/test/TestPreviewChunk.h', ]) qt_resource_files = qt.preprocess( @@ -1199,6 +1219,31 @@ svcore_data_fileio_test_exe = executable( win_subsystem: 'console' ) +tony_main_test_exe = executable( + 'test-tony-main', + tony_main_test_moc_files, + 'main/PreviewChunk.cpp', + 'main/test/tony-main-test.cpp', + include_directories: [ + 'main', + ], + dependencies: [ + svcore_dep, + qt_dep, + feature_dependencies, + dl_dep, + ], + cpp_args: [ + feature_defines, + general_defines, + ], + link_args: [ + feature_additional_libs, + general_link_args, + ], + win_subsystem: 'console' +) + test('svcore-base', svcore_base_test_exe) test('svcore-system', svcore_system_test_exe) test('svcore-data-model', svcore_data_model_test_exe) @@ -1206,6 +1251,7 @@ test('svcore-data-fileio', svcore_data_fileio_test_exe, args: [ '--testdir', meson.current_source_dir() / 'svcore/data/fileio/test' ]) +test('tony-main', tony_main_test_exe) summary({'prefix': get_option('prefix'), 'bindir': get_option('bindir'),