/* * Copyright (c) 2020-2022 Meltytech, LLC * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "proxymanager.h" #include "mltcontroller.h" #include "settings.h" #include "shotcut_mlt_properties.h" #include "jobqueue.h" #include "jobs/ffmpegjob.h" #include "jobs/qimagejob.h" #include "util.h" #include #include #include #include #include #include #include #include static const char *kProxySubfolder = "proxies"; static const char *kProxyVideoExtension = ".mp4"; static const char *kGoProProxyVideoExtension = ".LRV"; static const char *kProxyPendingVideoExtension = ".pending.mp4"; static const char *kProxyImageExtension = ".jpg"; static const char *kProxyPendingImageExtension = ".pending.jpg"; static const float kProxyResolutionRatio = 1.3f; static const int kFallbackProxyResolution = 540; static const QStringList kPixFmtsWithAlpha = {"pal8", "argb", "rgba", "abgr", "bgra", "yuva420p", "yuva422p", "yuva444p", "yuva420p9be", "yuva420p9le", "yuva422p9be", "yuva422p9le", "yuva444p9be", "yuva444p9le", "yuva420p10be", "yuva420p10le", "yuva422p10be", "yuva422p10le", "yuva444p10be", "yuva444p10le", "yuva420p16be", "yuva420p16le", "yuva422p16be", "yuva422p16le", "yuva444p16be", "yuva444p16le", "rgba64be", "rgba64le", "bgra64be", "bgra64le", "ya8", "ya16le", "ya16be", "gbrap", "gbrap16le", "gbrap16be", "ayuv64le", "ayuv64be", "gbrap12le", "gbrap12be", "gbrap10le", "gbrap10be", "gbrapf32be", "gbrapf32le", "yuva422p12be", "yuva422p12le", "yuva444p12be", "yuva444p12le" }; QDir ProxyManager::dir() { // Use project folder + "/proxies" if using project folder and enabled QDir dir(MLT.projectFolder()); if (!MLT.projectFolder().isEmpty() && dir.exists() && Settings.proxyUseProjectFolder()) { if (!dir.cd(kProxySubfolder)) { if (dir.mkdir(kProxySubfolder)) dir.cd(kProxySubfolder); } } else { // Otherwise, use app setting dir = QDir(Settings.proxyFolder()); } return dir; } QString ProxyManager::resource(Mlt::Service &producer) { QString resource = QString::fromUtf8(producer.get("resource")); if (producer.get_int(kIsProxyProperty) && producer.get(kOriginalResourceProperty)) { resource = QString::fromUtf8(producer.get(kOriginalResourceProperty)); } else if (!::qstrcmp(producer.get("mlt_service"), "timewarp")) { resource = QString::fromUtf8(producer.get("warp_resource")); } return resource; } void ProxyManager::generateVideoProxy(Mlt::Producer &producer, bool fullRange, ScanMode scanMode, const QPoint &aspectRatio, bool replace) { // Always regenerate per preview scaling or 540 if not specified QString resource = ProxyManager::resource(producer); QStringList args; QString hash = Util::getHash(producer); QString fileName = ProxyManager::dir().filePath(hash + kProxyPendingVideoExtension); QString filters; auto hwCodecs = Settings.encodeHardware(); QString hwFilters; // Touch file to make it in progress QFile file(fileName); file.open(QIODevice::WriteOnly); file.resize(0); file.close(); args << "-loglevel" << "verbose"; args << "-i" << resource; args << "-max_muxing_queue_size" << "9999"; // transcode all streams except data, subtitles, and attachments auto audioIndex = producer.property_exists(kDefaultAudioIndexProperty) ? producer.get_int( kDefaultAudioIndexProperty) : producer.get_int("audio_index"); if (producer.get_int("video_index") < audioIndex) { args << "-map" << "0:V?" << "-map" << "0:a?"; } else { args << "-map" << "0:a?" << "-map" << "0:V?"; } args << "-map_metadata" << "0" << "-ignore_unknown"; args << "-vf"; if (scanMode == Automatic) { filters = QString("yadif=deint=interlaced,"); } else if (scanMode != Progressive) { filters = QString("yadif=parity=%1,").arg(scanMode == InterlacedTopFieldFirst ? "tff" : "bff"); } filters += QString("scale=width=-2:height=%1").arg(resolution()); if (Settings.proxyUseHardware() && (hwCodecs.contains("hevc_vaapi") || hwCodecs.contains("h264_vaapi"))) { hwFilters = ",format=nv12,hwupload"; } if (fullRange) { args << filters + ":in_range=full:out_range=full" + hwFilters; args << "-color_range" << "jpeg"; } else { args << filters + ":in_range=mpeg:out_range=mpeg" + hwFilters; args << "-color_range" << "mpeg"; } switch (producer.get_int("meta.media.colorspace")) { case 601: if (producer.get_int("meta.media.height") == 576) { args << "-color_primaries" << "bt470bg"; args << "-color_trc" << "smpte170m"; args << "-colorspace" << "bt470bg"; } else { args << "-color_primaries" << "smpte170m"; args << "-color_trc" << "smpte170m"; args << "-colorspace" << "smpte170m"; } break; case 170: args << "-color_primaries" << "smpte170m"; args << "-color_trc" << "smpte170m"; args << "-colorspace" << "smpte170m"; break; case 240: args << "-color_primaries" << "smpte240m"; args << "-color_trc" << "smpte240m"; args << "-colorspace" << "smpte240m"; break; case 470: args << "-color_primaries" << "bt470bg"; args << "-color_trc" << "bt470bg"; args << "-colorspace" << "bt470bg"; break; default: args << "-color_primaries" << "bt709"; args << "-color_trc" << "bt709"; args << "-colorspace" << "bt709"; break; } if (!aspectRatio.isNull()) { args << "-aspect" << QString("%1:%2").arg(aspectRatio.x()).arg(aspectRatio.y()); } args << "-f" << "mp4" << "-codec:a" << "ac3" << "-b:a" << "256k"; args << "-pix_fmt" << "yuv420p"; if (Settings.proxyUseHardware()) { if (hwCodecs.contains("hevc_nvenc")) { args << "-codec:v" << "hevc_nvenc"; args << "-rc" << "constqp"; args << "-vglobal_quality" << "37"; } else if (hwCodecs.contains("hevc_qsv")) { args << "-load_plugin" << "hevc_hw"; args << "-codec:v" << "hevc_qsv"; args << "-q:v" << "36"; } else if (hwCodecs.contains("hevc_amf")) { args << "-codec:v" << "hevc_amf"; args << "-rc" << "1"; args << "-qp_i" << "32" << "-qp_p" << "32"; } else if (hwCodecs.contains("hevc_vaapi")) { args << "-init_hw_device" << "vaapi=vaapi0:" << "-filter_hw_device" << "vaapi0"; args << "-codec:v" << "hevc_vaapi"; args << "-qp" << "37"; } else if (hwCodecs.contains("h264_nvenc")) { args << "-codec:v" << "h264_nvenc"; args << "-rc" << "constqp"; args << "-vglobal_quality" << "37"; } else if (hwCodecs.contains("h264_vaapi")) { args << "-init_hw_device" << "vaapi=vaapi0:" << "-filter_hw_device" << "vaapi0"; args << "-codec:v" << "h264_vaapi"; args << "-qp" << "30"; } else if (hwCodecs.contains("hevc_videotoolbox")) { args << "-codec:v" << "hevc_videotoolbox"; args << "-b:v" << "2M"; } else if (hwCodecs.contains("h264_videotoolbox")) { args << "-codec:v" << "h264_videotoolbox"; args << "-b:v" << "2M"; } else if (hwCodecs.contains("h264_qsv")) { args << "-codec:v" << "h264_qsv"; args << "-q:v" << "36"; } else if (hwCodecs.contains("h264_amf")) { args << "-codec:v" << "h264_amf"; args << "-rc" << "1"; args << "-qp_i" << "32" << "-qp_p" << "32"; } } if (!args.contains("-codec:v")) { args << "-codec:v" << "libx264"; args << "-preset" << "veryfast"; args << "-crf" << "23"; } args << "-g" << "1" << "-bf" << "0"; args << "-y" << fileName; FfmpegJob *job = new FfmpegJob(fileName, args, true); job->setLabel(QObject::tr("Make proxy for %1").arg(Util::baseName(resource))); if (replace) { job->setPostJobAction(new ProxyReplacePostJobAction(resource, fileName, hash)); } else { job->setPostJobAction(new ProxyFinalizePostJobAction(resource, fileName)); } JOBS.add(job); } void ProxyManager::generateImageProxy(Mlt::Producer &producer, bool replace) { // Always regenerate per preview scaling or 540 if not specified QString resource = ProxyManager::resource(producer); QStringList args; QString hash = Util::getHash(producer); QString fileName = ProxyManager::dir().filePath(hash + kProxyPendingImageExtension); QString filters; // Touch file to make it in progress QFile file(fileName); file.open(QIODevice::WriteOnly); file.resize(0); file.close(); AbstractJob *job = new QImageJob(fileName, resource, resolution()); if (replace) { job->setPostJobAction(new ProxyReplacePostJobAction(resource, fileName, hash)); } else { job->setPostJobAction(new ProxyFinalizePostJobAction(resource, fileName)); } JOBS.add(job); } typedef QPair MltProperty; static void processProperties(QXmlStreamWriter &newXml, QVector &properties, const QString &root) { // Determine if this is a proxy resource bool isProxy = false; QString newResource; QString service; QString speed = "1"; for (const auto &p : properties) { if (p.first == kIsProxyProperty) { isProxy = true; } else if (p.first == kOriginalResourceProperty) { newResource = p.second; } else if (newResource.isEmpty() && p.first == "resource") { newResource = p.second; } else if (p.first == "mlt_service") { service = p.second; } else if (p.first == "warp_speed") { speed = p.second; } } QVector newProperties; QVector &propertiesRef = properties; if (isProxy) { // Filter the properties for (const auto &p : properties) { // Replace the resource property if proxy if (p.first == "resource") { // Convert to relative if (!root.isEmpty() && newResource.startsWith(root)) { newResource = newResource.mid(root.size()); } if (service == "timewarp") { newProperties << MltProperty(p.first, QString("%1:%2").arg(speed).arg(newResource)); } else { newProperties << MltProperty(p.first, newResource); } } else if (p.first == "warp_resource") { newProperties << MltProperty(p.first, newResource); // Remove special proxy and original resource properties } else if (p.first != kIsProxyProperty && p.first != kOriginalResourceProperty) { newProperties << MltProperty(p.first, p.second); } } propertiesRef = newProperties; } // Write all of the property elements for (const auto &p : propertiesRef) { newXml.writeStartElement("property"); newXml.writeAttribute("name", p.first); newXml.writeCharacters(p.second); newXml.writeEndElement(); } // Reset the saved properties properties.clear(); } bool ProxyManager::filterXML(QString &xmlString, QString root) { QString output; QXmlStreamReader xml(xmlString); QXmlStreamWriter newXml(&output); bool isPropertyElement = false; QVector properties; // This prevents processProperties() from mis-matching a resource path that begins with root // when it is converting to relative paths. if (!root.isEmpty() && root.endsWith('/')) { root.append('/'); } newXml.setAutoFormatting(true); newXml.setAutoFormattingIndent(2); while (!xml.atEnd()) { switch (xml.readNext()) { case QXmlStreamReader::Characters: if (!isPropertyElement) newXml.writeCharacters(xml.text().toString()); break; case QXmlStreamReader::Comment: newXml.writeComment(xml.text().toString()); break; case QXmlStreamReader::DTD: newXml.writeDTD(xml.text().toString()); break; case QXmlStreamReader::EntityReference: newXml.writeEntityReference(xml.name().toString()); break; case QXmlStreamReader::ProcessingInstruction: newXml.writeProcessingInstruction(xml.processingInstructionTarget().toString(), xml.processingInstructionData().toString()); break; case QXmlStreamReader::StartDocument: newXml.writeStartDocument(xml.documentVersion().toString(), xml.isStandaloneDocument()); break; case QXmlStreamReader::EndDocument: newXml.writeEndDocument(); break; case QXmlStreamReader::StartElement: { const QString element = xml.name().toString(); if (element == "property") { // Save each property element but do not output yet const QString name = xml.attributes().value("name").toString(); properties << MltProperty(name, xml.readElementText()); isPropertyElement = true; } else { // At the start of a non-property element isPropertyElement = false; processProperties(newXml, properties, root); // Write the new start element newXml.writeStartElement(xml.namespaceUri().toString(), element); for (const auto &a : xml.attributes()) { newXml.writeAttribute(a); } } break; } case QXmlStreamReader::EndElement: // At the end of a non-property element if (xml.name() != "property") { processProperties(newXml, properties, root); newXml.writeEndElement(); } break; default: break; } } // Useful for debugging // LOG_DEBUG() << output; if (!xml.hasError()) { xmlString = output; return true; } return false; } bool ProxyManager::fileExists(Mlt::Producer &producer) { QDir proxyDir(Settings.proxyFolder()); QDir projectDir(MLT.projectFolder()); QString service = QString::fromLatin1(producer.get("mlt_service")); QString fileName; if (service.startsWith("avformat")) { if (QFile::exists(GoProProxyFilePath(producer.get("resource")))) { return true; } fileName = Util::getHash(producer) + kProxyVideoExtension; } else if (isValidImage(producer)) { fileName = Util::getHash(producer) + kProxyImageExtension; } else { return false; } return (projectDir.cd(kProxySubfolder) && projectDir.exists(fileName)) || proxyDir.exists(fileName); } bool ProxyManager::filePending(Mlt::Producer &producer) { QDir proxyDir(Settings.proxyFolder()); QDir projectDir(MLT.projectFolder()); QString service = QString::fromLatin1(producer.get("mlt_service")); QString fileName; if (service.startsWith("avformat")) { fileName = Util::getHash(producer) + kProxyPendingVideoExtension; } else if (isValidImage(producer)) { fileName = Util::getHash(producer) + kProxyPendingImageExtension; } else { return false; } return (projectDir.cd(kProxySubfolder) && projectDir.exists(fileName)) || proxyDir.exists(fileName); } bool ProxyManager::isValidImage(Mlt::Producer &producer) { QString service = QString::fromLatin1(producer.get("mlt_service")); if ((service == "qimage" || service == "pixbuf") && !producer.get_int(kShotcutSequenceProperty)) { QImageReader reader; reader.setDecideFormatFromContent(true); reader.setFileName(ProxyManager::resource(producer)); return reader.imageCount() == 1 && !reader.read().hasAlphaChannel(); } return false; } bool ProxyManager::isValidVideo(Mlt::Producer producer) { QString service = QString::fromLatin1(producer.get("mlt_service")); int video_index = producer.get_int("video_index"); // video_index -1 means no video if (video_index < 0) return false; if (service == "avformat-novalidate") { producer = Mlt::Producer(MLT.profile(), resource(producer).toUtf8().constData()); service = QString::fromLatin1(producer.get("mlt_service")); producer.set("video_index", video_index); } if (service == "avformat") { QString key = QString("meta.media.%1.codec.pix_fmt").arg(video_index); QString pix_fmt = QString::fromLatin1(producer.get(key.toLatin1().constData())); // Cover art is usually 90000 fps and should not be proxied key = QString("meta.media.%1.codec.frame_rate").arg(video_index); QString frame_rate = producer.get(key.toLatin1().constData()); key = QString("meta.media.%1.codec.name").arg(video_index); QString codec_name = producer.get(key.toLatin1().constData()); bool coverArt = codec_name == "mjpeg" && frame_rate == "90000"; key = QString("meta.attr.%1.stream.alpha_mode.markup").arg(video_index); bool alpha_mode = producer.get_int(key.toLatin1().constData()); LOG_DEBUG() << "pix_fmt =" << pix_fmt << " codec.frame_rate =" << frame_rate << " alpha_mode =" << alpha_mode; return !kPixFmtsWithAlpha.contains(pix_fmt) && !alpha_mode && !coverArt; } return false; } // Returns true if the producer exists and was updated with proxy info bool ProxyManager::generateIfNotExists(Mlt::Producer &producer, bool replace) { if (Settings.proxyEnabled() && producer.is_valid() && !producer.get_int(kDisableProxyProperty) && !producer.get_int(kIsProxyProperty)) { if (ProxyManager::fileExists(producer)) { QString service = QString::fromLatin1(producer.get("mlt_service")); QDir projectDir(MLT.projectFolder()); QString fileName; if (service.startsWith("avformat")) { auto gopro = GoProProxyFilePath(producer.get("resource")); if (QFile::exists(gopro)) { producer.set(kIsProxyProperty, 1); producer.set(kOriginalResourceProperty, producer.get("resource")); producer.set("resource", gopro.toUtf8().constData()); return true; } else { fileName = Util::getHash(producer) + kProxyVideoExtension; } } else if (isValidImage(producer)) { fileName = Util::getHash(producer) + kProxyImageExtension; } else { return false; } producer.set(kIsProxyProperty, 1); producer.set(kOriginalResourceProperty, producer.get("resource")); if (projectDir.exists(fileName)) { ::utime(projectDir.filePath(fileName).toUtf8().constData(), nullptr); producer.set("resource", projectDir.filePath(fileName).toUtf8().constData()); } else { QDir proxyDir(Settings.proxyFolder()); ::utime(proxyDir.filePath(fileName).toUtf8().constData(), nullptr); producer.set("resource", proxyDir.filePath(fileName).toUtf8().constData()); } return true; } else if (!filePending(producer)) { if (isValidVideo(producer)) { // Tag this producer so we do not try to generate proxy again in this session delete producer.get_frame(); auto threshold = qRound(kProxyResolutionRatio * resolution()); LOG_DEBUG() << producer.get_int("meta.media.width") << "x" << producer.get_int("meta.media.height") << "threshold" << threshold; if (producer.get_int("meta.media.width") > threshold && producer.get_int("meta.media.height") > threshold) { ProxyManager::generateVideoProxy(producer, MLT.fullRange(producer), Automatic, QPoint(), replace); } } else if (isValidImage(producer)) { // Tag this producer so we do not try to generate proxy again in this session delete producer.get_frame(); auto threshold = qRound(kProxyResolutionRatio * resolution()); LOG_DEBUG() << producer.get_int("meta.media.width") << "x" << producer.get_int("meta.media.height") << "threshold" << threshold; if (producer.get_int("meta.media.width") > threshold && producer.get_int("meta.media.height") > threshold) { ProxyManager::generateImageProxy(producer, replace); } } } } return false; } const char *ProxyManager::videoFilenameExtension() { return kProxyVideoExtension; } const char *ProxyManager::pendingVideoExtension() { return kProxyPendingVideoExtension; } const char *ProxyManager::imageFilenameExtension() { return kProxyImageExtension; } const char *ProxyManager::pendingImageExtension() { return kProxyImageExtension; } int ProxyManager::resolution() { return Settings.playerPreviewScale() ? Settings.playerPreviewScale() : kFallbackProxyResolution; } class FindNonProxyProducersParser : public Mlt::Parser { private: QString m_hash; QList m_producers; public: FindNonProxyProducersParser() : Mlt::Parser() {} QList &producers() { return m_producers; } int on_start_filter(Mlt::Filter *) { return 0; } int on_start_producer(Mlt::Producer *producer) { if (!producer->parent().get_int(kIsProxyProperty)) m_producers << Mlt::Producer(producer); return 0; } int on_end_producer(Mlt::Producer *) { return 0; } int on_start_playlist(Mlt::Playlist *) { return 0; } int on_end_playlist(Mlt::Playlist *) { return 0; } int on_start_tractor(Mlt::Tractor *) { return 0; } int on_end_tractor(Mlt::Tractor *) { return 0; } int on_start_multitrack(Mlt::Multitrack *) { return 0; } int on_end_multitrack(Mlt::Multitrack *) { return 0; } int on_start_track() { return 0; } int on_end_track() { return 0; } int on_end_filter(Mlt::Filter *) { return 0; } int on_start_transition(Mlt::Transition *) { return 0; } int on_end_transition(Mlt::Transition *) { return 0; } int on_start_chain(Mlt::Chain *chain) { if (!chain->parent().get_int(kIsProxyProperty)) m_producers << Mlt::Producer(chain); return 0; } int on_end_chain(Mlt::Chain *) { return 0; } int on_start_link(Mlt::Link *) { return 0; } int on_end_link(Mlt::Link *) { return 0; } }; void ProxyManager::generateIfNotExistsAll(Mlt::Producer &producer) { FindNonProxyProducersParser parser; parser.start(producer); for (auto &clip : parser.producers()) { generateIfNotExists(clip, false /* replace */); } } bool ProxyManager::removePending() { bool foundAny = false; QDir dir(MLT.projectFolder()); if (!MLT.projectFolder().isEmpty() && dir.exists()) { dir.cd(kProxySubfolder); } else { dir = QDir(Settings.proxyFolder()); } if (dir.exists()) { dir.setNameFilters(QStringList() << "*.pending.*"); dir.setFilter(QDir::Files | QDir::NoDotAndDotDot | QDir::Writable); for (const auto &s : dir.entryList()) { LOG_INFO() << "removing" << dir.filePath(s); foundAny |= QFile::remove(dir.filePath(s)); } } //TODO if any pending remove, let user know and offer to regenerate? return foundAny; } QString ProxyManager::GoProProxyFilePath(const QString &resource) { auto fi = QFileInfo(resource); auto base = fi.baseName(); base = "GL" + base.mid(2); auto result = fi.absoluteDir().filePath(base + kGoProProxyVideoExtension); LOG_DEBUG() << result; return result; }