diff --git a/meson_options.txt b/meson_options.txt index 2104469e3..866067042 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -91,3 +91,8 @@ option('v4l2', value : 'auto', description : 'Compile the V4L2 compatibility layer', deprecated : {'true': 'enabled', 'false': 'disabled'}) + +option('rpi_nn_awb', + type : 'feature', + value : 'disabled', + description : 'Enable Raspberry Pi neural network AWB') diff --git a/src/ipa/rpi/controller/meson.build b/src/ipa/rpi/controller/meson.build index dde4ac127..e6d0f719d 100644 --- a/src/ipa/rpi/controller/meson.build +++ b/src/ipa/rpi/controller/meson.build @@ -10,6 +10,7 @@ rpi_ipa_controller_sources = files([ 'rpi/agc_channel.cpp', 'rpi/alsc.cpp', 'rpi/awb.cpp', + 'rpi/awb_bayes.cpp', 'rpi/black_level.cpp', 'rpi/cac.cpp', 'rpi/ccm.cpp', @@ -31,6 +32,22 @@ rpi_ipa_controller_deps = [ libcamera_private, ] +if not get_option('rpi_nn_awb').disabled() + tflite_dep = dependency('tensorflow-lite', required : false) + if not tflite_dep.found() + # This may not have a pkg-config file, so try and find it directly + cc = meson.get_compiler('cpp') + tflite_dep = cc.find_library('tensorflow-lite', required : get_option('rpi_nn_awb')) + endif + + if tflite_dep.found() + rpi_ipa_controller_sources += files([ + 'rpi/awb_nn.cpp', + ]) + rpi_ipa_controller_deps += tflite_dep + endif +endif + rpi_ipa_controller_lib = static_library('rpi_ipa_controller', rpi_ipa_controller_sources, include_directories : libipa_includes, dependencies : rpi_ipa_controller_deps) diff --git a/src/ipa/rpi/controller/rpi/awb.cpp b/src/ipa/rpi/controller/rpi/awb.cpp index 365b595ff..de5fa59bf 100644 --- a/src/ipa/rpi/controller/rpi/awb.cpp +++ b/src/ipa/rpi/controller/rpi/awb.cpp @@ -1,20 +1,14 @@ /* SPDX-License-Identifier: BSD-2-Clause */ /* - * Copyright (C) 2019, Raspberry Pi Ltd + * Copyright (C) 2025, Raspberry Pi Ltd * * AWB control algorithm */ - -#include -#include -#include - -#include +#include "awb.h" #include "../lux_status.h" #include "alsc_status.h" -#include "awb.h" using namespace RPiController; using namespace libcamera; @@ -23,39 +17,6 @@ LOG_DEFINE_CATEGORY(RPiAwb) constexpr double kDefaultCT = 4500.0; -#define NAME "rpi.awb" - -/* - * todo - the locking in this algorithm needs some tidying up as has been done - * elsewhere (ALSC and AGC). - */ - -int AwbMode::read(const libcamera::YamlObject ¶ms) -{ - auto value = params["lo"].get(); - if (!value) - return -EINVAL; - ctLo = *value; - - value = params["hi"].get(); - if (!value) - return -EINVAL; - ctHi = *value; - - return 0; -} - -int AwbPrior::read(const libcamera::YamlObject ¶ms) -{ - auto value = params["lux"].get(); - if (!value) - return -EINVAL; - lux = *value; - - prior = params["prior"].get(ipa::Pwl{}); - return prior.empty() ? -EINVAL : 0; -} - static int readCtCurve(ipa::Pwl &ctR, ipa::Pwl &ctB, const libcamera::YamlObject ¶ms) { if (params.size() % 3) { @@ -92,11 +53,25 @@ static int readCtCurve(ipa::Pwl &ctR, ipa::Pwl &ctB, const libcamera::YamlObject return 0; } +int AwbMode::read(const libcamera::YamlObject ¶ms) +{ + auto value = params["lo"].get(); + if (!value) + return -EINVAL; + ctLo = *value; + + value = params["hi"].get(); + if (!value) + return -EINVAL; + ctHi = *value; + + return 0; +} + int AwbConfig::read(const libcamera::YamlObject ¶ms) { int ret; - bayes = params["bayes"].get(1); framePeriod = params["frame_period"].get(10); startupFrames = params["startup_frames"].get(10); convergenceFrames = params["convergence_frames"].get(3); @@ -111,23 +86,6 @@ int AwbConfig::read(const libcamera::YamlObject ¶ms) ctBInverse = ctB.inverse().first; } - if (params.contains("priors")) { - for (const auto &p : params["priors"].asList()) { - AwbPrior prior; - ret = prior.read(p); - if (ret) - return ret; - if (!priors.empty() && prior.lux <= priors.back().lux) { - LOG(RPiAwb, Error) << "AwbConfig: Prior must be ordered in increasing lux value"; - return -EINVAL; - } - priors.push_back(prior); - } - if (priors.empty()) { - LOG(RPiAwb, Error) << "AwbConfig: no AWB priors configured"; - return -EINVAL; - } - } if (params.contains("modes")) { for (const auto &[key, value] : params["modes"].asDict()) { ret = modes[key].read(value); @@ -142,13 +100,10 @@ int AwbConfig::read(const libcamera::YamlObject ¶ms) } } - minPixels = params["min_pixels"].get(16.0); - minG = params["min_G"].get(32); - minRegions = params["min_regions"].get(10); deltaLimit = params["delta_limit"].get(0.2); - coarseStep = params["coarse_step"].get(0.2); transversePos = params["transverse_pos"].get(0.01); transverseNeg = params["transverse_neg"].get(0.01); + if (transversePos <= 0 || transverseNeg <= 0) { LOG(RPiAwb, Error) << "AwbConfig: transverse_pos/neg must be > 0"; return -EINVAL; @@ -157,29 +112,21 @@ int AwbConfig::read(const libcamera::YamlObject ¶ms) sensitivityR = params["sensitivity_r"].get(1.0); sensitivityB = params["sensitivity_b"].get(1.0); - if (bayes) { - if (ctR.empty() || ctB.empty() || priors.empty() || - defaultMode == nullptr) { - LOG(RPiAwb, Warning) - << "Bayesian AWB mis-configured - switch to Grey method"; - bayes = false; - } - } - whitepointR = params["whitepoint_r"].get(0.0); - whitepointB = params["whitepoint_b"].get(0.0); - if (bayes == false) + if (hasCtCurve() && defaultMode != nullptr) { + greyWorld = false; + } else { + greyWorld = true; sensitivityR = sensitivityB = 1.0; /* nor do sensitivities make any sense */ - /* - * The biasProportion parameter adds a small proportion of the counted - * pixles to a region biased to the biasCT colour temperature. - * - * A typical value for biasProportion would be between 0.05 to 0.1. - */ - biasProportion = params["bias_proportion"].get(0.0); - biasCT = params["bias_ct"].get(kDefaultCT); + } + return 0; } +bool AwbConfig::hasCtCurve() const +{ + return !ctR.empty() && !ctB.empty(); +} + Awb::Awb(Controller *controller) : AwbAlgorithm(controller) { @@ -199,16 +146,6 @@ Awb::~Awb() asyncThread_.join(); } -char const *Awb::name() const -{ - return NAME; -} - -int Awb::read(const libcamera::YamlObject ¶ms) -{ - return config_.read(params); -} - void Awb::initialise() { frameCount_ = framePhase_ = 0; @@ -217,7 +154,7 @@ void Awb::initialise() * just in case the first few frames don't have anything meaningful in * them. */ - if (!config_.ctR.empty() && !config_.ctB.empty()) { + if (!config_.greyWorld) { syncResults_.temperatureK = config_.ctR.domain().clamp(4000); syncResults_.gainR = 1.0 / config_.ctR.eval(syncResults_.temperatureK); syncResults_.gainG = 1.0; @@ -282,7 +219,7 @@ void Awb::setManualGains(double manualR, double manualB) syncResults_.gainR = prevSyncResults_.gainR = manualR_; syncResults_.gainG = prevSyncResults_.gainG = 1.0; syncResults_.gainB = prevSyncResults_.gainB = manualB_; - if (config_.bayes) { + if (!config_.greyWorld) { /* Also estimate the best corresponding colour temperature from the curves. */ double ctR = config_.ctRInverse.eval(config_.ctRInverse.domain().clamp(1 / manualR_)); double ctB = config_.ctBInverse.eval(config_.ctBInverse.domain().clamp(1 / manualB_)); @@ -294,7 +231,7 @@ void Awb::setManualGains(double manualR, double manualB) void Awb::setColourTemperature(double temperatureK) { - if (!config_.bayes) { + if (config_.greyWorld) { LOG(RPiAwb, Warning) << "AWB uncalibrated - cannot set colour temperature"; return; } @@ -433,10 +370,10 @@ void Awb::asyncFunc() } } -static void generateStats(std::vector &zones, - StatisticsPtr &stats, double minPixels, - double minG, Metadata &globalMetadata, - double biasProportion, double biasCtR, double biasCtB) +void Awb::generateStats(std::vector &zones, + StatisticsPtr &stats, double minPixels, + double minG, Metadata &globalMetadata, + double biasProportion, double biasCtR, double biasCtB) { std::scoped_lock l(globalMetadata); @@ -450,9 +387,9 @@ static void generateStats(std::vector &zones, zone.R = region.val.rSum / region.counted; zone.B = region.val.bSum / region.counted; /* - * Add some bias samples to allow the search to tend to a - * bias CT in failure cases. - */ + * Add some bias samples to allow the search to tend to a + * bias CT in failure cases. + */ const unsigned int proportion = biasProportion * region.counted; zone.R += proportion * biasCtR; zone.B += proportion * biasCtB; @@ -469,29 +406,7 @@ static void generateStats(std::vector &zones, } } -void Awb::prepareStats() -{ - zones_.clear(); - /* - * LSC has already been applied to the stats in this pipeline, so stop - * any LSC compensation. We also ignore config_.fast in this version. - */ - const double biasCtR = config_.bayes ? config_.ctR.eval(config_.biasCT) : 0; - const double biasCtB = config_.bayes ? config_.ctB.eval(config_.biasCT) : 0; - generateStats(zones_, statistics_, config_.minPixels, - config_.minG, getGlobalMetadata(), - config_.biasProportion, biasCtR, biasCtB); - /* - * apply sensitivities, so values appear to come from our "canonical" - * sensor. - */ - for (auto &zone : zones_) { - zone.R *= config_.sensitivityR; - zone.B *= config_.sensitivityB; - } -} - -double Awb::computeDelta2Sum(double gainR, double gainB) +double Awb::computeDelta2Sum(double gainR, double gainB, double whitepointR, double whitepointB) { /* * Compute the sum of the squared colour error (non-greyness) as it @@ -499,8 +414,8 @@ double Awb::computeDelta2Sum(double gainR, double gainB) */ double delta2Sum = 0; for (auto &z : zones_) { - double deltaR = gainR * z.R - 1 - config_.whitepointR; - double deltaB = gainB * z.B - 1 - config_.whitepointB; + double deltaR = gainR * z.R - 1 - whitepointR; + double deltaB = gainB * z.B - 1 - whitepointB; double delta2 = deltaR * deltaR + deltaB * deltaB; /* LOG(RPiAwb, Debug) << "deltaR " << deltaR << " deltaB " << deltaB << " delta2 " << delta2; */ delta2 = std::min(delta2, config_.deltaLimit); @@ -509,39 +424,14 @@ double Awb::computeDelta2Sum(double gainR, double gainB) return delta2Sum; } -ipa::Pwl Awb::interpolatePrior() +double Awb::interpolateQuadatric(libcamera::ipa::Pwl::Point const &a, + libcamera::ipa::Pwl::Point const &b, + libcamera::ipa::Pwl::Point const &c) { /* - * Interpolate the prior log likelihood function for our current lux - * value. - */ - if (lux_ <= config_.priors.front().lux) - return config_.priors.front().prior; - else if (lux_ >= config_.priors.back().lux) - return config_.priors.back().prior; - else { - int idx = 0; - /* find which two we lie between */ - while (config_.priors[idx + 1].lux < lux_) - idx++; - double lux0 = config_.priors[idx].lux, - lux1 = config_.priors[idx + 1].lux; - return ipa::Pwl::combine(config_.priors[idx].prior, - config_.priors[idx + 1].prior, - [&](double /*x*/, double y0, double y1) { - return y0 + (y1 - y0) * - (lux_ - lux0) / (lux1 - lux0); - }); - } -} - -static double interpolateQuadatric(ipa::Pwl::Point const &a, ipa::Pwl::Point const &b, - ipa::Pwl::Point const &c) -{ - /* - * Given 3 points on a curve, find the extremum of the function in that - * interval by fitting a quadratic. - */ + * Given 3 points on a curve, find the extremum of the function in that + * interval by fitting a quadratic. + */ const double eps = 1e-3; ipa::Pwl::Point ca = c - a, ba = b - a; double denominator = 2 * (ba.y() * ca.x() - ca.y() * ba.x()); @@ -554,180 +444,6 @@ static double interpolateQuadatric(ipa::Pwl::Point const &a, ipa::Pwl::Point con return a.y() < c.y() - eps ? a.x() : (c.y() < a.y() - eps ? c.x() : b.x()); } -double Awb::coarseSearch(ipa::Pwl const &prior) -{ - points_.clear(); /* assume doesn't deallocate memory */ - size_t bestPoint = 0; - double t = mode_->ctLo; - int spanR = 0, spanB = 0; - /* Step down the CT curve evaluating log likelihood. */ - while (true) { - double r = config_.ctR.eval(t, &spanR); - double b = config_.ctB.eval(t, &spanB); - double gainR = 1 / r, gainB = 1 / b; - double delta2Sum = computeDelta2Sum(gainR, gainB); - double priorLogLikelihood = prior.eval(prior.domain().clamp(t)); - double finalLogLikelihood = delta2Sum - priorLogLikelihood; - LOG(RPiAwb, Debug) - << "t: " << t << " gain R " << gainR << " gain B " - << gainB << " delta2_sum " << delta2Sum - << " prior " << priorLogLikelihood << " final " - << finalLogLikelihood; - points_.push_back(ipa::Pwl::Point({ t, finalLogLikelihood })); - if (points_.back().y() < points_[bestPoint].y()) - bestPoint = points_.size() - 1; - if (t == mode_->ctHi) - break; - /* for even steps along the r/b curve scale them by the current t */ - t = std::min(t + t / 10 * config_.coarseStep, mode_->ctHi); - } - t = points_[bestPoint].x(); - LOG(RPiAwb, Debug) << "Coarse search found CT " << t; - /* - * We have the best point of the search, but refine it with a quadratic - * interpolation around its neighbours. - */ - if (points_.size() > 2) { - unsigned long bp = std::min(bestPoint, points_.size() - 2); - bestPoint = std::max(1UL, bp); - t = interpolateQuadatric(points_[bestPoint - 1], - points_[bestPoint], - points_[bestPoint + 1]); - LOG(RPiAwb, Debug) - << "After quadratic refinement, coarse search has CT " - << t; - } - return t; -} - -void Awb::fineSearch(double &t, double &r, double &b, ipa::Pwl const &prior) -{ - int spanR = -1, spanB = -1; - config_.ctR.eval(t, &spanR); - config_.ctB.eval(t, &spanB); - double step = t / 10 * config_.coarseStep * 0.1; - int nsteps = 5; - double rDiff = config_.ctR.eval(t + nsteps * step, &spanR) - - config_.ctR.eval(t - nsteps * step, &spanR); - double bDiff = config_.ctB.eval(t + nsteps * step, &spanB) - - config_.ctB.eval(t - nsteps * step, &spanB); - ipa::Pwl::Point transverse({ bDiff, -rDiff }); - if (transverse.length2() < 1e-6) - return; - /* - * unit vector orthogonal to the b vs. r function (pointing outwards - * with r and b increasing) - */ - transverse = transverse / transverse.length(); - double bestLogLikelihood = 0, bestT = 0, bestR = 0, bestB = 0; - double transverseRange = config_.transverseNeg + config_.transversePos; - const int maxNumDeltas = 12; - /* a transverse step approximately every 0.01 r/b units */ - int numDeltas = floor(transverseRange * 100 + 0.5) + 1; - numDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas); - /* - * Step down CT curve. March a bit further if the transverse range is - * large. - */ - nsteps += numDeltas; - for (int i = -nsteps; i <= nsteps; i++) { - double tTest = t + i * step; - double priorLogLikelihood = - prior.eval(prior.domain().clamp(tTest)); - double rCurve = config_.ctR.eval(tTest, &spanR); - double bCurve = config_.ctB.eval(tTest, &spanB); - /* x will be distance off the curve, y the log likelihood there */ - ipa::Pwl::Point points[maxNumDeltas]; - int bestPoint = 0; - /* Take some measurements transversely *off* the CT curve. */ - for (int j = 0; j < numDeltas; j++) { - points[j][0] = -config_.transverseNeg + - (transverseRange * j) / (numDeltas - 1); - ipa::Pwl::Point rbTest = ipa::Pwl::Point({ rCurve, bCurve }) + - transverse * points[j].x(); - double rTest = rbTest.x(), bTest = rbTest.y(); - double gainR = 1 / rTest, gainB = 1 / bTest; - double delta2Sum = computeDelta2Sum(gainR, gainB); - points[j][1] = delta2Sum - priorLogLikelihood; - LOG(RPiAwb, Debug) - << "At t " << tTest << " r " << rTest << " b " - << bTest << ": " << points[j].y(); - if (points[j].y() < points[bestPoint].y()) - bestPoint = j; - } - /* - * We have NUM_DELTAS points transversely across the CT curve, - * now let's do a quadratic interpolation for the best result. - */ - bestPoint = std::max(1, std::min(bestPoint, numDeltas - 2)); - ipa::Pwl::Point rbTest = ipa::Pwl::Point({ rCurve, bCurve }) + - transverse * interpolateQuadatric(points[bestPoint - 1], - points[bestPoint], - points[bestPoint + 1]); - double rTest = rbTest.x(), bTest = rbTest.y(); - double gainR = 1 / rTest, gainB = 1 / bTest; - double delta2Sum = computeDelta2Sum(gainR, gainB); - double finalLogLikelihood = delta2Sum - priorLogLikelihood; - LOG(RPiAwb, Debug) - << "Finally " - << tTest << " r " << rTest << " b " << bTest << ": " - << finalLogLikelihood - << (finalLogLikelihood < bestLogLikelihood ? " BEST" : ""); - if (bestT == 0 || finalLogLikelihood < bestLogLikelihood) - bestLogLikelihood = finalLogLikelihood, - bestT = tTest, bestR = rTest, bestB = bTest; - } - t = bestT, r = bestR, b = bestB; - LOG(RPiAwb, Debug) - << "Fine search found t " << t << " r " << r << " b " << b; -} - -void Awb::awbBayes() -{ - /* - * May as well divide out G to save computeDelta2Sum from doing it over - * and over. - */ - for (auto &z : zones_) - z.R = z.R / (z.G + 1), z.B = z.B / (z.G + 1); - /* - * Get the current prior, and scale according to how many zones are - * valid... not entirely sure about this. - */ - ipa::Pwl prior = interpolatePrior(); - prior *= zones_.size() / (double)(statistics_->awbRegions.numRegions()); - prior.map([](double x, double y) { - LOG(RPiAwb, Debug) << "(" << x << "," << y << ")"; - }); - double t = coarseSearch(prior); - double r = config_.ctR.eval(t); - double b = config_.ctB.eval(t); - LOG(RPiAwb, Debug) - << "After coarse search: r " << r << " b " << b << " (gains r " - << 1 / r << " b " << 1 / b << ")"; - /* - * Not entirely sure how to handle the fine search yet. Mostly the - * estimated CT is already good enough, but the fine search allows us to - * wander transverely off the CT curve. Under some illuminants, where - * there may be more or less green light, this may prove beneficial, - * though I probably need more real datasets before deciding exactly how - * this should be controlled and tuned. - */ - fineSearch(t, r, b, prior); - LOG(RPiAwb, Debug) - << "After fine search: r " << r << " b " << b << " (gains r " - << 1 / r << " b " << 1 / b << ")"; - /* - * Write results out for the main thread to pick up. Remember to adjust - * the gains from the ones that the "canonical sensor" would require to - * the ones needed by *this* sensor. - */ - asyncResults_.temperatureK = t; - asyncResults_.gainR = 1.0 / r * config_.sensitivityR; - asyncResults_.gainG = 1.0; - asyncResults_.gainB = 1.0 / b * config_.sensitivityB; -} - void Awb::awbGrey() { LOG(RPiAwb, Debug) << "Grey world AWB"; @@ -765,32 +481,3 @@ void Awb::awbGrey() asyncResults_.gainG = 1.0; asyncResults_.gainB = gainB; } - -void Awb::doAwb() -{ - prepareStats(); - LOG(RPiAwb, Debug) << "Valid zones: " << zones_.size(); - if (zones_.size() > config_.minRegions) { - if (config_.bayes) - awbBayes(); - else - awbGrey(); - LOG(RPiAwb, Debug) - << "CT found is " - << asyncResults_.temperatureK - << " with gains r " << asyncResults_.gainR - << " and b " << asyncResults_.gainB; - } - /* - * we're done with these; we may as well relinquish our hold on the - * pointer. - */ - statistics_.reset(); -} - -/* Register algorithm with the system. */ -static Algorithm *create(Controller *controller) -{ - return (Algorithm *)new Awb(controller); -} -static RegisterAlgorithm reg(NAME, &create); diff --git a/src/ipa/rpi/controller/rpi/awb.h b/src/ipa/rpi/controller/rpi/awb.h index 2fb912541..e4f1e94bb 100644 --- a/src/ipa/rpi/controller/rpi/awb.h +++ b/src/ipa/rpi/controller/rpi/awb.h @@ -1,42 +1,33 @@ /* SPDX-License-Identifier: BSD-2-Clause */ /* - * Copyright (C) 2019, Raspberry Pi Ltd + * Copyright (C) 2025, Raspberry Pi Ltd * * AWB control algorithm */ #pragma once -#include #include +#include #include -#include - #include "../awb_algorithm.h" #include "../awb_status.h" -#include "../statistics.h" - #include "libipa/pwl.h" namespace RPiController { -/* Control algorithm to perform AWB calculations. */ - struct AwbMode { int read(const libcamera::YamlObject ¶ms); double ctLo; /* low CT value for search */ double ctHi; /* high CT value for search */ }; -struct AwbPrior { - int read(const libcamera::YamlObject ¶ms); - double lux; /* lux level */ - libcamera::ipa::Pwl prior; /* maps CT to prior log likelihood for this lux level */ -}; - struct AwbConfig { - AwbConfig() : defaultMode(nullptr) {} + AwbConfig() + : defaultMode(nullptr) {} int read(const libcamera::YamlObject ¶ms); + bool hasCtCurve() const; + /* Only repeat the AWB calculation every "this many" frames */ uint16_t framePeriod; /* number of initial frames for which speed taken as 1.0 (maximum) */ @@ -47,27 +38,13 @@ struct AwbConfig { libcamera::ipa::Pwl ctB; /* function maps CT to b (= B/G) */ libcamera::ipa::Pwl ctRInverse; /* inverse of ctR */ libcamera::ipa::Pwl ctBInverse; /* inverse of ctB */ - /* table of illuminant priors at different lux levels */ - std::vector priors; + /* AWB "modes" (determines the search range) */ std::map modes; AwbMode *defaultMode; /* mode used if no mode selected */ - /* - * minimum proportion of pixels counted within AWB region for it to be - * "useful" - */ - double minPixels; - /* minimum G value of those pixels, to be regarded a "useful" */ - uint16_t minG; - /* - * number of AWB regions that must be "useful" in order to do the AWB - * calculation - */ - uint32_t minRegions; + /* clamp on colour error term (so as not to penalise non-grey excessively) */ double deltaLimit; - /* step size control in coarse search */ - double coarseStep; /* how far to wander off CT curve towards "more purple" */ double transversePos; /* how far to wander off CT curve towards "more green" */ @@ -82,14 +59,8 @@ struct AwbConfig { * sensor's B/G) */ double sensitivityB; - /* The whitepoint (which we normally "aim" for) can be moved. */ - double whitepointR; - double whitepointB; - bool bayes; /* use Bayesian algorithm */ - /* proportion of counted samples to add for the search bias */ - double biasProportion; - /* CT target for the search bias */ - double biasCT; + + bool greyWorld; /* don't use the ct curve when in grey world mode */ }; class Awb : public AwbAlgorithm @@ -97,9 +68,7 @@ class Awb : public AwbAlgorithm public: Awb(Controller *controller = NULL); ~Awb(); - char const *name() const override; - void initialise() override; - int read(const libcamera::YamlObject ¶ms) override; + virtual void initialise() override; unsigned int getConvergenceFrames() const override; void initialValues(double &gainR, double &gainB) override; void setMode(std::string const &name) override; @@ -110,6 +79,11 @@ class Awb : public AwbAlgorithm void switchMode(CameraMode const &cameraMode, Metadata *metadata) override; void prepare(Metadata *imageMetadata) override; void process(StatisticsPtr &stats, Metadata *imageMetadata) override; + + static double interpolateQuadatric(libcamera::ipa::Pwl::Point const &a, + libcamera::ipa::Pwl::Point const &b, + libcamera::ipa::Pwl::Point const &c); + struct RGB { RGB(double r = 0, double g = 0, double b = 0) : R(r), G(g), B(b) @@ -123,10 +97,30 @@ class Awb : public AwbAlgorithm } }; -private: - bool isAutoEnabled() const; +protected: /* configuration is read-only, and available to both threads */ AwbConfig config_; + /* + * The following are for the asynchronous thread to use, though the main + * thread can set/reset them if the async thread is known to be idle: + */ + std::vector zones_; + StatisticsPtr statistics_; + double lux_; + AwbMode *mode_; + AwbStatus asyncResults_; + + virtual void doAwb() = 0; + virtual void prepareStats() = 0; + double computeDelta2Sum(double gainR, double gainB, double whitepointR, double whitepointB); + void awbGrey(); + static void generateStats(std::vector &zones, + StatisticsPtr &stats, double minPixels, + double minG, Metadata &globalMetadata, + double biasProportion, double biasCtR, double biasCtB); + +private: + bool isAutoEnabled() const; std::thread asyncThread_; void asyncFunc(); /* asynchronous thread function */ std::mutex mutex_; @@ -142,9 +136,9 @@ class Awb : public AwbAlgorithm bool asyncAbort_; /* - * The following are only for the synchronous thread to use: - * for sync thread to note its has asked async thread to run - */ + * The following are only for the synchronous thread to use: + * for sync thread to note its has asked async thread to run + */ bool asyncStarted_; /* counts up to framePeriod before restarting the async thread */ int framePhase_; @@ -152,6 +146,7 @@ class Awb : public AwbAlgorithm AwbStatus syncResults_; AwbStatus prevSyncResults_; std::string modeName_; + /* * The following are for the asynchronous thread to use, though the main * thread can set/reset them if the async thread is known to be idle: @@ -159,20 +154,6 @@ class Awb : public AwbAlgorithm void restartAsync(StatisticsPtr &stats, double lux); /* copy out the results from the async thread so that it can be restarted */ void fetchAsyncResults(); - StatisticsPtr statistics_; - AwbMode *mode_; - double lux_; - AwbStatus asyncResults_; - void doAwb(); - void awbBayes(); - void awbGrey(); - void prepareStats(); - double computeDelta2Sum(double gainR, double gainB); - libcamera::ipa::Pwl interpolatePrior(); - double coarseSearch(libcamera::ipa::Pwl const &prior); - void fineSearch(double &t, double &r, double &b, libcamera::ipa::Pwl const &prior); - std::vector zones_; - std::vector points_; /* manual r setting */ double manualR_; /* manual b setting */ @@ -196,4 +177,4 @@ static inline Awb::RGB operator*(Awb::RGB const &rgb, double d) return d * rgb; } -} /* namespace RPiController */ +} // namespace RPiController diff --git a/src/ipa/rpi/controller/rpi/awb_bayes.cpp b/src/ipa/rpi/controller/rpi/awb_bayes.cpp new file mode 100644 index 000000000..09233cec8 --- /dev/null +++ b/src/ipa/rpi/controller/rpi/awb_bayes.cpp @@ -0,0 +1,444 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * + * AWB control algorithm + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "../awb_algorithm.h" +#include "../awb_status.h" +#include "../lux_status.h" +#include "../statistics.h" +#include "libipa/pwl.h" + +#include "alsc_status.h" +#include "awb.h" + +using namespace libcamera; + +LOG_DECLARE_CATEGORY(RPiAwb) + +constexpr double kDefaultCT = 4500.0; + +#define NAME "rpi.awb" + +/* + * todo - the locking in this algorithm needs some tidying up as has been done + * elsewhere (ALSC and AGC). + */ + +namespace RPiController { + +struct AwbPrior { + int read(const libcamera::YamlObject ¶ms); + double lux; /* lux level */ + libcamera::ipa::Pwl prior; /* maps CT to prior log likelihood for this lux level */ +}; + +struct AwbBayesConfig { + AwbBayesConfig() {} + int read(const libcamera::YamlObject ¶ms, AwbConfig &config); + /* table of illuminant priors at different lux levels */ + std::vector priors; + /* + * minimum proportion of pixels counted within AWB region for it to be + * "useful" + */ + double minPixels; + /* minimum G value of those pixels, to be regarded a "useful" */ + uint16_t minG; + /* + * number of AWB regions that must be "useful" in order to do the AWB + * calculation + */ + uint32_t minRegions; + /* step size control in coarse search */ + double coarseStep; + /* The whitepoint (which we normally "aim" for) can be moved. */ + double whitepointR; + double whitepointB; + bool bayes; /* use Bayesian algorithm */ + /* proportion of counted samples to add for the search bias */ + double biasProportion; + /* CT target for the search bias */ + double biasCT; +}; + +class AwbBayes : public Awb +{ +public: + AwbBayes(Controller *controller = NULL); + ~AwbBayes(); + char const *name() const override; + int read(const libcamera::YamlObject ¶ms) override; + +protected: + void prepareStats() override; + void doAwb() override; + +private: + AwbBayesConfig bayesConfig_; + void awbBayes(); + libcamera::ipa::Pwl interpolatePrior(); + double coarseSearch(libcamera::ipa::Pwl const &prior); + void fineSearch(double &t, double &r, double &b, libcamera::ipa::Pwl const &prior); + std::vector points_; +}; + +int AwbPrior::read(const libcamera::YamlObject ¶ms) +{ + auto value = params["lux"].get(); + if (!value) + return -EINVAL; + lux = *value; + + prior = params["prior"].get(ipa::Pwl{}); + return prior.empty() ? -EINVAL : 0; +} + +int AwbBayesConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config) +{ + int ret; + + bayes = params["bayes"].get(1); + + if (params.contains("priors")) { + for (const auto &p : params["priors"].asList()) { + AwbPrior prior; + ret = prior.read(p); + if (ret) + return ret; + if (!priors.empty() && prior.lux <= priors.back().lux) { + LOG(RPiAwb, Error) << "AwbConfig: Prior must be ordered in increasing lux value"; + return -EINVAL; + } + priors.push_back(prior); + } + if (priors.empty()) { + LOG(RPiAwb, Error) << "AwbConfig: no AWB priors configured"; + return -EINVAL; + } + } + + minPixels = params["min_pixels"].get(16.0); + minG = params["min_G"].get(32); + minRegions = params["min_regions"].get(10); + coarseStep = params["coarse_step"].get(0.2); + + if (bayes) { + if (!config.hasCtCurve() || priors.empty() || + config.defaultMode == nullptr) { + LOG(RPiAwb, Warning) + << "Bayesian AWB mis-configured - switch to Grey method"; + bayes = false; + } + } + whitepointR = params["whitepoint_r"].get(0.0); + whitepointB = params["whitepoint_b"].get(0.0); + if (bayes == false) { + config.sensitivityR = config.sensitivityB = 1.0; /* nor do sensitivities make any sense */ + config.greyWorld = true; /* prevent the ct curve being used in manual mode */ + } + /* + * The biasProportion parameter adds a small proportion of the counted + * pixles to a region biased to the biasCT colour temperature. + * + * A typical value for biasProportion would be between 0.05 to 0.1. + */ + biasProportion = params["bias_proportion"].get(0.0); + biasCT = params["bias_ct"].get(kDefaultCT); + return 0; +} + +AwbBayes::AwbBayes(Controller *controller) + : Awb(controller) +{ +} + +AwbBayes::~AwbBayes() +{ +} + +char const *AwbBayes::name() const +{ + return NAME; +} + +int AwbBayes::read(const libcamera::YamlObject ¶ms) +{ + int ret; + + ret = config_.read(params); + if (ret) + return ret; + + ret = bayesConfig_.read(params, config_); + if (ret) + return ret; + + return 0; +} + +void AwbBayes::prepareStats() +{ + zones_.clear(); + /* + * LSC has already been applied to the stats in this pipeline, so stop + * any LSC compensation. We also ignore config_.fast in this version. + */ + const double biasCtR = bayesConfig_.bayes ? config_.ctR.eval(bayesConfig_.biasCT) : 0; + const double biasCtB = bayesConfig_.bayes ? config_.ctB.eval(bayesConfig_.biasCT) : 0; + generateStats(zones_, statistics_, bayesConfig_.minPixels, + bayesConfig_.minG, getGlobalMetadata(), + bayesConfig_.biasProportion, biasCtR, biasCtB); + /* + * apply sensitivities, so values appear to come from our "canonical" + * sensor. + */ + for (auto &zone : zones_) { + zone.R *= config_.sensitivityR; + zone.B *= config_.sensitivityB; + } +} + +ipa::Pwl AwbBayes::interpolatePrior() +{ + /* + * Interpolate the prior log likelihood function for our current lux + * value. + */ + if (lux_ <= bayesConfig_.priors.front().lux) + return bayesConfig_.priors.front().prior; + else if (lux_ >= bayesConfig_.priors.back().lux) + return bayesConfig_.priors.back().prior; + else { + int idx = 0; + /* find which two we lie between */ + while (bayesConfig_.priors[idx + 1].lux < lux_) + idx++; + double lux0 = bayesConfig_.priors[idx].lux, + lux1 = bayesConfig_.priors[idx + 1].lux; + return ipa::Pwl::combine(bayesConfig_.priors[idx].prior, + bayesConfig_.priors[idx + 1].prior, + [&](double /*x*/, double y0, double y1) { + return y0 + (y1 - y0) * + (lux_ - lux0) / (lux1 - lux0); + }); + } +} + +double AwbBayes::coarseSearch(ipa::Pwl const &prior) +{ + points_.clear(); /* assume doesn't deallocate memory */ + size_t bestPoint = 0; + double t = mode_->ctLo; + int spanR = 0, spanB = 0; + /* Step down the CT curve evaluating log likelihood. */ + while (true) { + double r = config_.ctR.eval(t, &spanR); + double b = config_.ctB.eval(t, &spanB); + double gainR = 1 / r, gainB = 1 / b; + double delta2Sum = computeDelta2Sum(gainR, gainB, bayesConfig_.whitepointR, bayesConfig_.whitepointB); + double priorLogLikelihood = prior.eval(prior.domain().clamp(t)); + double finalLogLikelihood = delta2Sum - priorLogLikelihood; + LOG(RPiAwb, Debug) + << "t: " << t << " gain R " << gainR << " gain B " + << gainB << " delta2_sum " << delta2Sum + << " prior " << priorLogLikelihood << " final " + << finalLogLikelihood; + points_.push_back(ipa::Pwl::Point({ t, finalLogLikelihood })); + if (points_.back().y() < points_[bestPoint].y()) + bestPoint = points_.size() - 1; + if (t == mode_->ctHi) + break; + /* for even steps along the r/b curve scale them by the current t */ + t = std::min(t + t / 10 * bayesConfig_.coarseStep, mode_->ctHi); + } + t = points_[bestPoint].x(); + LOG(RPiAwb, Debug) << "Coarse search found CT " << t; + /* + * We have the best point of the search, but refine it with a quadratic + * interpolation around its neighbours. + */ + if (points_.size() > 2) { + unsigned long bp = std::min(bestPoint, points_.size() - 2); + bestPoint = std::max(1UL, bp); + t = interpolateQuadatric(points_[bestPoint - 1], + points_[bestPoint], + points_[bestPoint + 1]); + LOG(RPiAwb, Debug) + << "After quadratic refinement, coarse search has CT " + << t; + } + return t; +} + +void AwbBayes::fineSearch(double &t, double &r, double &b, ipa::Pwl const &prior) +{ + int spanR = -1, spanB = -1; + config_.ctR.eval(t, &spanR); + config_.ctB.eval(t, &spanB); + double step = t / 10 * bayesConfig_.coarseStep * 0.1; + int nsteps = 5; + double rDiff = config_.ctR.eval(t + nsteps * step, &spanR) - + config_.ctR.eval(t - nsteps * step, &spanR); + double bDiff = config_.ctB.eval(t + nsteps * step, &spanB) - + config_.ctB.eval(t - nsteps * step, &spanB); + ipa::Pwl::Point transverse({ bDiff, -rDiff }); + if (transverse.length2() < 1e-6) + return; + /* + * unit vector orthogonal to the b vs. r function (pointing outwards + * with r and b increasing) + */ + transverse = transverse / transverse.length(); + double bestLogLikelihood = 0, bestT = 0, bestR = 0, bestB = 0; + double transverseRange = config_.transverseNeg + config_.transversePos; + const int maxNumDeltas = 12; + /* a transverse step approximately every 0.01 r/b units */ + int numDeltas = floor(transverseRange * 100 + 0.5) + 1; + numDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas); + /* + * Step down CT curve. March a bit further if the transverse range is + * large. + */ + nsteps += numDeltas; + for (int i = -nsteps; i <= nsteps; i++) { + double tTest = t + i * step; + double priorLogLikelihood = + prior.eval(prior.domain().clamp(tTest)); + double rCurve = config_.ctR.eval(tTest, &spanR); + double bCurve = config_.ctB.eval(tTest, &spanB); + /* x will be distance off the curve, y the log likelihood there */ + ipa::Pwl::Point points[maxNumDeltas]; + int bestPoint = 0; + /* Take some measurements transversely *off* the CT curve. */ + for (int j = 0; j < numDeltas; j++) { + points[j][0] = -config_.transverseNeg + + (transverseRange * j) / (numDeltas - 1); + ipa::Pwl::Point rbTest = ipa::Pwl::Point({ rCurve, bCurve }) + + transverse * points[j].x(); + double rTest = rbTest.x(), bTest = rbTest.y(); + double gainR = 1 / rTest, gainB = 1 / bTest; + double delta2Sum = computeDelta2Sum(gainR, gainB, bayesConfig_.whitepointR, bayesConfig_.whitepointB); + points[j][1] = delta2Sum - priorLogLikelihood; + LOG(RPiAwb, Debug) + << "At t " << tTest << " r " << rTest << " b " + << bTest << ": " << points[j].y(); + if (points[j].y() < points[bestPoint].y()) + bestPoint = j; + } + /* + * We have NUM_DELTAS points transversely across the CT curve, + * now let's do a quadratic interpolation for the best result. + */ + bestPoint = std::max(1, std::min(bestPoint, numDeltas - 2)); + ipa::Pwl::Point rbTest = ipa::Pwl::Point({ rCurve, bCurve }) + + transverse * interpolateQuadatric(points[bestPoint - 1], + points[bestPoint], + points[bestPoint + 1]); + double rTest = rbTest.x(), bTest = rbTest.y(); + double gainR = 1 / rTest, gainB = 1 / bTest; + double delta2Sum = computeDelta2Sum(gainR, gainB, bayesConfig_.whitepointR, bayesConfig_.whitepointB); + double finalLogLikelihood = delta2Sum - priorLogLikelihood; + LOG(RPiAwb, Debug) + << "Finally " + << tTest << " r " << rTest << " b " << bTest << ": " + << finalLogLikelihood + << (finalLogLikelihood < bestLogLikelihood ? " BEST" : ""); + if (bestT == 0 || finalLogLikelihood < bestLogLikelihood) + bestLogLikelihood = finalLogLikelihood, + bestT = tTest, bestR = rTest, bestB = bTest; + } + t = bestT, r = bestR, b = bestB; + LOG(RPiAwb, Debug) + << "Fine search found t " << t << " r " << r << " b " << b; +} + +void AwbBayes::awbBayes() +{ + /* + * May as well divide out G to save computeDelta2Sum from doing it over + * and over. + */ + for (auto &z : zones_) + z.R = z.R / (z.G + 1), z.B = z.B / (z.G + 1); + /* + * Get the current prior, and scale according to how many zones are + * valid... not entirely sure about this. + */ + ipa::Pwl prior = interpolatePrior(); + prior *= zones_.size() / (double)(statistics_->awbRegions.numRegions()); + prior.map([](double x, double y) { + LOG(RPiAwb, Debug) << "(" << x << "," << y << ")"; + }); + double t = coarseSearch(prior); + double r = config_.ctR.eval(t); + double b = config_.ctB.eval(t); + LOG(RPiAwb, Debug) + << "After coarse search: r " << r << " b " << b << " (gains r " + << 1 / r << " b " << 1 / b << ")"; + /* + * Not entirely sure how to handle the fine search yet. Mostly the + * estimated CT is already good enough, but the fine search allows us to + * wander transverely off the CT curve. Under some illuminants, where + * there may be more or less green light, this may prove beneficial, + * though I probably need more real datasets before deciding exactly how + * this should be controlled and tuned. + */ + fineSearch(t, r, b, prior); + LOG(RPiAwb, Debug) + << "After fine search: r " << r << " b " << b << " (gains r " + << 1 / r << " b " << 1 / b << ")"; + /* + * Write results out for the main thread to pick up. Remember to adjust + * the gains from the ones that the "canonical sensor" would require to + * the ones needed by *this* sensor. + */ + asyncResults_.temperatureK = t; + asyncResults_.gainR = 1.0 / r * config_.sensitivityR; + asyncResults_.gainG = 1.0; + asyncResults_.gainB = 1.0 / b * config_.sensitivityB; +} + +void AwbBayes::doAwb() +{ + prepareStats(); + LOG(RPiAwb, Debug) << "Valid zones: " << zones_.size(); + if (zones_.size() > bayesConfig_.minRegions) { + if (bayesConfig_.bayes) + awbBayes(); + else + awbGrey(); + LOG(RPiAwb, Debug) + << "CT found is " + << asyncResults_.temperatureK + << " with gains r " << asyncResults_.gainR + << " and b " << asyncResults_.gainB; + } + /* + * we're done with these; we may as well relinquish our hold on the + * pointer. + */ + statistics_.reset(); +} + +/* Register algorithm with the system. */ +static Algorithm *create(Controller *controller) +{ + return (Algorithm *)new AwbBayes(controller); +} +static RegisterAlgorithm reg(NAME, &create); + +} /* namespace RPiController */ diff --git a/src/ipa/rpi/controller/rpi/awb_nn.cpp b/src/ipa/rpi/controller/rpi/awb_nn.cpp new file mode 100644 index 000000000..01d8b7126 --- /dev/null +++ b/src/ipa/rpi/controller/rpi/awb_nn.cpp @@ -0,0 +1,533 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2025, Raspberry Pi Ltd + * + * AWB control algorithm using neural network + */ + +#include +#include + +#include +#include + +#include +#include +#include + +#include "../awb_algorithm.h" +#include "../awb_status.h" +#include "../lux_status.h" +#include "libipa/pwl.h" + +#include "alsc_status.h" +#include "awb.h" + +using namespace libcamera; + +LOG_DECLARE_CATEGORY(RPiAwb) + +constexpr double kDefaultCT = 4500.0; + +#define NAME "rpi.nn.awb" + +namespace RPiController { + +struct AwbNNConfig { + AwbNNConfig() {} + int read(const libcamera::YamlObject ¶ms, AwbConfig &config); + + /* An empty model will check default locations for model.tflite */ + std::string model; + int numOutputs; + float minTemp; + float maxTemp; + + bool enableNn; + + /* CCM matrix for 5000K temperature */ + double ccm[9]; +}; + +class AwbNN : public Awb +{ +public: + AwbNN(Controller *controller = NULL); + ~AwbNN(); + char const *name() const override; + void initialise() override; + int read(const libcamera::YamlObject ¶ms) override; + +protected: + void doAwb() override; + void prepareStats() override; + +private: + bool isAutoEnabled() const; + AwbNNConfig nnConfig_; + void transverseSearch(double t, double &r, double &b); + RGB processZone(RGB zone, float lux, float red_gain, float blue_gain); + void awbNN(); + void loadModel(); + + std::unique_ptr model_; + std::unique_ptr interpreter_; +}; + +int AwbNNConfig::read(const libcamera::YamlObject ¶ms, AwbConfig &config) +{ + model = params["model"].get(""); + numOutputs = params["num_outputs"].get(10); + minTemp = params["min_temp"].get(2800.0); + maxTemp = params["max_temp"].get(7600.0); + + for (int i = 0; i < 9; i++) + ccm[i] = params["ccm"][i].get(0.0); + + enableNn = params["enable_nn"].get(1); + + if (enableNn) { + if (!config.hasCtCurve()) { + LOG(RPiAwb, Error) << "CT curve not specified"; + enableNn = false; + } + + if (!model.empty() && model.find(".tflite") == std::string::npos) { + LOG(RPiAwb, Error) << "Model must be a .tflite file"; + enableNn = false; + } + + bool validCcm = true; + for (int i = 0; i < 9; i++) + if (ccm[i] == 0.0) + validCcm = false; + + if (!validCcm) { + LOG(RPiAwb, Error) << "CCM not specified or invalid"; + enableNn = false; + } + + if (!enableNn) { + LOG(RPiAwb, Warning) << "Neural Network AWB mis-configured - switch to Grey method"; + } + } + + if (!enableNn) { + config.sensitivityR = config.sensitivityB = 1.0; + config.greyWorld = false; + } + + return 0; +} + +AwbNN::AwbNN(Controller *controller) + : Awb(controller) +{ +} + +AwbNN::~AwbNN() +{ +} + +char const *AwbNN::name() const +{ + return NAME; +} + +int AwbNN::read(const libcamera::YamlObject ¶ms) +{ + int ret; + + ret = config_.read(params); + if (ret) + return ret; + + ret = nnConfig_.read(params, config_); + if (ret) + return ret; + + return 0; +} + +void AwbNN::loadModel() +{ + LOG(RPiAwb, Debug) << "Loading AWB model " << nnConfig_.model; + std::string modelPath = "/ipa/rpi/pisp/awb_model.tflite"; + if (nnConfig_.model.empty()) { + std::string root = utils::libcameraSourcePath(); + if (!root.empty()) { + modelPath = root + modelPath; + } else { + modelPath = LIBCAMERA_DATA_DIR + modelPath; + } + + if (!File::exists(modelPath)) { + LOG(RPiAwb, Error) << "No model file found in standard locations"; + nnConfig_.enableNn = false; + return; + } + } else { + modelPath = nnConfig_.model; + } + + LOG(RPiAwb, Debug) << "Attempting to load model from: " << modelPath; + + model_ = tflite::FlatBufferModel::BuildFromFile(modelPath.c_str()); + + if (!model_) { + LOG(RPiAwb, Error) << "Failed to load model from " << modelPath; + nnConfig_.enableNn = false; + return; + } + + tflite::MutableOpResolver resolver; + tflite::ops::builtin::BuiltinOpResolver builtin_resolver; + resolver.AddAll(builtin_resolver); + tflite::InterpreterBuilder(*model_, resolver)(&interpreter_); + if (!interpreter_) { + LOG(RPiAwb, Error) << "Failed to build interpreter for model " << nnConfig_.model; + nnConfig_.enableNn = false; + return; + } + + interpreter_->AllocateTensors(); + TfLiteTensor *inputTensor = interpreter_->input_tensor(0); + TfLiteTensor *outputTensor = interpreter_->output_tensor(0); + if (!inputTensor || !outputTensor) { + LOG(RPiAwb, Error) << "Model missing input or output tensor"; + nnConfig_.enableNn = false; + return; + } + + const int expectedInputSize = 32 * 32 * 3; + int actualInputSize = 1; + for (int i = 0; i < inputTensor->dims->size; i++) { + actualInputSize *= inputTensor->dims->data[i]; + } + + if (actualInputSize != expectedInputSize) { + LOG(RPiAwb, Error) << "Model input tensor size mismatch. Expected: " + << expectedInputSize << ", Got: " << actualInputSize; + nnConfig_.enableNn = false; + return; + } + + int actualOutputSize = 1; + for (int i = 0; i < outputTensor->dims->size; i++) { + actualOutputSize *= outputTensor->dims->data[i]; + } + + if (actualOutputSize != nnConfig_.numOutputs) { + LOG(RPiAwb, Error) << "Model output tensor size mismatch. Expected: " + << nnConfig_.numOutputs << ", Got: " << actualOutputSize; + nnConfig_.enableNn = false; + return; + } + + if (inputTensor->type != kTfLiteFloat32 || outputTensor->type != kTfLiteFloat32) { + LOG(RPiAwb, Error) << "Model input and output tensor must be float32"; + nnConfig_.enableNn = false; + return; + } + + LOG(RPiAwb, Info) << "Model loaded successfully from " << modelPath; + LOG(RPiAwb, Debug) << "Model validation successful - Input: " << actualInputSize + << " floats, Output: " << actualOutputSize << " floats"; +} + +void AwbNN::initialise() +{ + Awb::initialise(); + + if (nnConfig_.enableNn) { + loadModel(); + if (!nnConfig_.enableNn) { + LOG(RPiAwb, Warning) << "Neural Network AWB failed to load - switch to Grey method"; + config_.greyWorld = false; + config_.sensitivityR = config_.sensitivityB = 1.0; + } + } +} + +void AwbNN::prepareStats() +{ + zones_.clear(); + /* + * LSC has already been applied to the stats in this pipeline, so stop + * any LSC compensation. We also ignore config_.fast in this version. + */ + generateStats(zones_, statistics_, 0.0, 0.0, getGlobalMetadata(), 0.0, 0.0, 0.0); + /* + * apply sensitivities, so values appear to come from our "canonical" + * sensor. + */ + for (auto &zone : zones_) { + zone.R *= config_.sensitivityR; + zone.B *= config_.sensitivityB; + } +} + +void AwbNN::transverseSearch(double t, double &r, double &b) +{ + int spanR = -1, spanB = -1; + config_.ctR.eval(t, &spanR); + config_.ctB.eval(t, &spanB); + + const int diff = 10; + double rDiff = config_.ctR.eval(t + diff, &spanR) - + config_.ctR.eval(t - diff, &spanR); + double bDiff = config_.ctB.eval(t + diff, &spanB) - + config_.ctB.eval(t - diff, &spanB); + + ipa::Pwl::Point transverse({ bDiff, -rDiff }); + if (transverse.length2() < 1e-6) + return; + + transverse = transverse / transverse.length(); + double transverseRange = config_.transverseNeg + config_.transversePos; + const int maxNumDeltas = 12; + int numDeltas = floor(transverseRange * 100 + 0.5) + 1; + numDeltas = numDeltas < 3 ? 3 : (numDeltas > maxNumDeltas ? maxNumDeltas : numDeltas); + + ipa::Pwl::Point points[maxNumDeltas]; + int bestPoint = 0; + + for (int i = 0; i < numDeltas; i++) { + points[i][0] = -config_.transverseNeg + + (transverseRange * i) / (numDeltas - 1); + ipa::Pwl::Point rbTest = ipa::Pwl::Point({ r, b }) + + transverse * points[i].x(); + double rTest = rbTest.x(), bTest = rbTest.y(); + double gainR = 1 / rTest, gainB = 1 / bTest; + double delta2Sum = computeDelta2Sum(gainR, gainB, 0.0, 0.0); + points[i][1] = delta2Sum; + if (points[i].y() < points[bestPoint].y()) + bestPoint = i; + } + + bestPoint = std::clamp(bestPoint, 1, numDeltas - 2); + ipa::Pwl::Point rbBest = ipa::Pwl::Point({ r, b }) + + transverse * interpolateQuadatric(points[bestPoint - 1], + points[bestPoint], + points[bestPoint + 1]); + double rBest = rbBest.x(), bBest = rbBest.y(); + + r = rBest, b = bBest; +} + +static double getElementPadded(const std::vector &array, int i, int j) +{ + int iPadded = std::clamp(i, 0, 31); + int jPadded = std::clamp(j, 0, 31); + return array[iPadded * 32 + jPadded]; +} + +static std::vector filter(const std::vector &rgb) +{ + /* + * Decreases the difference between the colour channels when it is higher + * than nearby pixels. + */ + const int totalPixels = 32 * 32; + + std::vector newRgb(rgb.size()); + + std::vector rDiff(totalPixels); + std::vector bDiff(totalPixels); + + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 32; j++) { + int idx = i * 32 + j; + const AwbNN::RGB &pixel = rgb[idx]; + rDiff[idx] = pixel.R - pixel.G; + bDiff[idx] = pixel.B - pixel.G; + } + } + + std::vector minRDiff(totalPixels); + std::vector minBDiff(totalPixels); + + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 32; j++) { + int idx = i * 32 + j; + + double minAbsR = std::numeric_limits::max(); + double minAbsB = std::numeric_limits::max(); + + double selectedR = 0.0; + double selectedB = 0.0; + + for (int di = -1; di <= 1; di++) { + for (int dj = -1; dj <= 1; dj++) { + double r = getElementPadded(rDiff, i + di, j + dj); + double b = getElementPadded(bDiff, i + di, j + dj); + + if (std::abs(r) < minAbsR) { + minAbsR = std::abs(r); + selectedR = r; + } + + if (std::abs(b) < minAbsB) { + minAbsB = std::abs(b); + selectedB = b; + } + } + } + minRDiff[idx] = selectedR; + minBDiff[idx] = selectedB; + } + } + + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 32; j++) { + int idx = i * 32 + j; + + const AwbNN::RGB &original_pixel = rgb[idx]; + double g = original_pixel.G; + AwbNN::RGB newPixel(g + minRDiff[idx], g, g + minBDiff[idx]); + newRgb[idx] = newPixel; + } + } + + return newRgb; +} + +AwbNN::RGB AwbNN::processZone(AwbNN::RGB zone, float lux, float redGain, float blueGain) +{ + /* + * Renders the pixel at 5000K temperature then takes the log of the + * r/g and b/g ratios. The log is then scaled to 0-1 and clamped. + * These are then combined with the lux to get the input for the. + * neural network. + */ + RGB zoneGains = zone; + + zoneGains.R *= redGain; + zoneGains.G *= 1.0; + zoneGains.B *= blueGain; + + RGB zoneCcm = zoneGains; + + zoneCcm.R = nnConfig_.ccm[0] * zoneGains.R + nnConfig_.ccm[1] * zoneGains.G + nnConfig_.ccm[2] * zoneGains.B; + zoneCcm.G = nnConfig_.ccm[3] * zoneGains.R + nnConfig_.ccm[4] * zoneGains.G + nnConfig_.ccm[5] * zoneGains.B; + zoneCcm.B = nnConfig_.ccm[6] * zoneGains.R + nnConfig_.ccm[7] * zoneGains.G + nnConfig_.ccm[8] * zoneGains.B; + + RGB zoneLog; + + double green = std::clamp(zoneCcm.G, 0.1, 1.0); + + double rRatio = zoneCcm.R / green; + double bRatio = zoneCcm.B / green; + + rRatio = std::clamp(rRatio, 0.01, 1.0); + bRatio = std::clamp(bRatio, 0.01, 1.0); + + zoneLog.R = std::log(rRatio); + zoneLog.G = std::log(lux) / 10; + zoneLog.B = std::log(bRatio); + + /* + * Clamp the log values so the ratio is at most about 2.71828 (e) + */ + zoneLog.R += 1.0; + zoneLog.B += 1.0; + zoneLog.R /= 2.0; + zoneLog.B /= 2.0; + + zoneLog.R = std::clamp(zoneLog.R, 0.0, 1.0); + zoneLog.G = std::clamp(zoneLog.G, 0.0, 1.0); + zoneLog.B = std::clamp(zoneLog.B, 0.0, 1.0); + + return zoneLog; +} + +static float interpolateOutput(float *outputData, int numOutputs, float minTemp, float maxTemp) +{ + float totalValue = 0.0; + float totalWeight = 0.0; + + for (int i = 0; i < numOutputs; i++) { + float temp = minTemp + i * (maxTemp - minTemp) / (numOutputs - 1); + float weight = outputData[i]; + weight *= weight; + totalValue += temp * weight; + totalWeight += weight; + } + + return totalValue / totalWeight; +} + +void AwbNN::awbNN() +{ + float *inputData = interpreter_->typed_input_tensor(0); + + float redGain = 1.0 / config_.ctR.eval(5000); + float blueGain = 1.0 / config_.ctB.eval(5000); + + std::vector zones = filter(zones_); + + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 32; j++) { + int zoneIdx = i * 32 + j; + + RGB processedZone = processZone(zones[zoneIdx] * (1.0 / 65535), lux_, redGain, blueGain); + + int baseIdx = (i * 32 + j) * 3; + + inputData[baseIdx + 0] = static_cast(processedZone.R); + inputData[baseIdx + 1] = static_cast(processedZone.G); + inputData[baseIdx + 2] = static_cast(processedZone.B); + } + } + + TfLiteStatus status = interpreter_->Invoke(); + if (status != kTfLiteOk) { + LOG(RPiAwb, Error) << "Model inference failed with status: " << status; + return; + } + + float *outputData = interpreter_->typed_output_tensor(0); + + LOG(RPiAwb, Debug) << "Output data: "; + for (int i = 0; i < nnConfig_.numOutputs; i++) { + LOG(RPiAwb, Debug) << "Output " << i << ": " << outputData[i]; + } + + double t = interpolateOutput(outputData, nnConfig_.numOutputs, nnConfig_.minTemp, nnConfig_.maxTemp); + + LOG(RPiAwb, Debug) << "Interpolated temperature: " << t; + + t = std::clamp(t, mode_->ctLo, mode_->ctHi); + + double r = config_.ctR.eval(t); + double b = config_.ctB.eval(t); + + transverseSearch(t, r, b); + + LOG(RPiAwb, Debug) << "After transverse search: Temperature: " << t << " Red gain: " << 1.0 / r << " Blue gain: " << 1.0 / b; + + asyncResults_.temperatureK = t; + asyncResults_.gainR = 1.0 / r * config_.sensitivityR; + asyncResults_.gainG = 1.0; + asyncResults_.gainB = 1.0 / b * config_.sensitivityB; +} + +void AwbNN::doAwb() +{ + prepareStats(); + if (zones_.size() == 1024 && nnConfig_.enableNn) { + awbNN(); + } else { + awbGrey(); + } + statistics_.reset(); +} + +/* Register algorithm with the system. */ +static Algorithm *create(Controller *controller) +{ + return (Algorithm *)new AwbNN(controller); +} +static RegisterAlgorithm reg(NAME, &create); + +} /* namespace RPiController */ diff --git a/src/ipa/rpi/pisp/data/awb_model.tflite b/src/ipa/rpi/pisp/data/awb_model.tflite new file mode 100644 index 000000000..2d471f9c9 Binary files /dev/null and b/src/ipa/rpi/pisp/data/awb_model.tflite differ diff --git a/src/ipa/rpi/pisp/data/imx219.json b/src/ipa/rpi/pisp/data/imx219.json index 3d076548b..316afe12a 100644 --- a/src/ipa/rpi/pisp/data/imx219.json +++ b/src/ipa/rpi/pisp/data/imx219.json @@ -195,6 +195,69 @@ "transverse_neg": 0.034 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2860.0, 0.9514, 0.4156, + 2960.0, 0.9289, 0.4372, + 3603.0, 0.8305, 0.5251, + 4650.0, 0.6756, 0.6433, + 5858.0, 0.6193, 0.6807, + 7580.0, 0.5019, 0.7495 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.03392, + "transverse_neg": 0.034, + "ccm": + [ + 2.2229345364238413, -0.7596721523178808, -0.46326238410596027, + -0.6834893874172185, 2.7118816887417223, -1.02839940397351, + -0.2613746357615894, -0.668015927152318, 1.9293905629139072 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -1187,6 +1250,6 @@ "rpi.sync": { } - } + } ] } \ No newline at end of file diff --git a/src/ipa/rpi/pisp/data/imx296.json b/src/ipa/rpi/pisp/data/imx296.json index 7fabb48d8..333b41910 100644 --- a/src/ipa/rpi/pisp/data/imx296.json +++ b/src/ipa/rpi/pisp/data/imx296.json @@ -194,6 +194,68 @@ "transverse_neg": 0.02154 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2875.0, 0.4699, 0.3209, + 3610.0, 0.4089, 0.4265, + 4640.0, 0.3281, 0.5417, + 5912.0, 0.2992, 0.5771, + 7630.0, 0.2285, 0.6524 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.01783, + "transverse_neg": 0.02154, + "ccm": + [ + 2.1073753846153847, -0.8054946153846154, -0.30188076923076923, + -0.43306999999999995, 2.162828076923077, -0.7297680769230768, + -0.126655, -0.5027626923076922, 1.6294176923076922 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -1194,6 +1256,6 @@ "rpi.sync": { } - } + } ] } \ No newline at end of file diff --git a/src/ipa/rpi/pisp/data/imx296_16mm.json b/src/ipa/rpi/pisp/data/imx296_16mm.json index 372d1ded0..b4c5389dc 100644 --- a/src/ipa/rpi/pisp/data/imx296_16mm.json +++ b/src/ipa/rpi/pisp/data/imx296_16mm.json @@ -194,6 +194,68 @@ "transverse_neg": 0.02154 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2875.0, 0.4699, 0.3209, + 3610.0, 0.4089, 0.4265, + 4640.0, 0.3281, 0.5417, + 5912.0, 0.2992, 0.5771, + 7630.0, 0.2285, 0.6524 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.01783, + "transverse_neg": 0.02154, + "ccm": + [ + 2.1073753846153847, -0.8054946153846154, -0.30188076923076923, + -0.43306999999999995, 2.162828076923077, -0.7297680769230768, + -0.126655, -0.5027626923076922, 1.6294176923076922 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -1247,6 +1309,6 @@ "rpi.sync": { } - } + } ] } \ No newline at end of file diff --git a/src/ipa/rpi/pisp/data/imx296_6mm.json b/src/ipa/rpi/pisp/data/imx296_6mm.json index 7b946c851..035ef18b4 100644 --- a/src/ipa/rpi/pisp/data/imx296_6mm.json +++ b/src/ipa/rpi/pisp/data/imx296_6mm.json @@ -194,6 +194,68 @@ "transverse_neg": 0.02154 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2875.0, 0.4699, 0.3209, + 3610.0, 0.4089, 0.4265, + 4640.0, 0.3281, 0.5417, + 5912.0, 0.2992, 0.5771, + 7630.0, 0.2285, 0.6524 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.01783, + "transverse_neg": 0.02154, + "ccm": + [ + 2.1073753846153847, -0.8054946153846154, -0.30188076923076923, + -0.43306999999999995, 2.162828076923077, -0.7297680769230768, + -0.126655, -0.5027626923076922, 1.6294176923076922 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -1247,6 +1309,6 @@ "rpi.sync": { } - } + } ] } \ No newline at end of file diff --git a/src/ipa/rpi/pisp/data/imx477.json b/src/ipa/rpi/pisp/data/imx477.json index 314ec639c..91d226336 100644 --- a/src/ipa/rpi/pisp/data/imx477.json +++ b/src/ipa/rpi/pisp/data/imx477.json @@ -195,6 +195,69 @@ "transverse_neg": 0.02255 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2850.0, 0.4307, 0.3957, + 2960.0, 0.4159, 0.4313, + 3580.0, 0.3771, 0.5176, + 4559.0, 0.3031, 0.6573, + 5881.0, 0.2809, 0.6942, + 7600.0, 0.2263, 0.7762 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.02634, + "transverse_neg": 0.02255, + "ccm": + [ + 2.1643743343419066, -0.972589984871407, -0.19177768532526474, + -0.3769567095310136, 2.0993768608169443, -0.722416815431165, + -0.11786965204236007, -0.4893621633888049, 1.607231815431165 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { diff --git a/src/ipa/rpi/pisp/data/imx477_16mm.json b/src/ipa/rpi/pisp/data/imx477_16mm.json index 8b94d1cf5..a4ac8e58b 100644 --- a/src/ipa/rpi/pisp/data/imx477_16mm.json +++ b/src/ipa/rpi/pisp/data/imx477_16mm.json @@ -195,6 +195,69 @@ "transverse_neg": 0.02255 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2850.0, 0.4307, 0.3957, + 2960.0, 0.4159, 0.4313, + 3580.0, 0.3771, 0.5176, + 4559.0, 0.3031, 0.6573, + 5881.0, 0.2809, 0.6942, + 7600.0, 0.2263, 0.7762 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.02634, + "transverse_neg": 0.02255, + "ccm": + [ + 2.1643743343419066, -0.972589984871407, -0.19177768532526474, + -0.3769567095310136, 2.0993768608169443, -0.722416815431165, + -0.11786965204236007, -0.4893621633888049, 1.607231815431165 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -1240,6 +1303,6 @@ "rpi.sync": { } - } + } ] } \ No newline at end of file diff --git a/src/ipa/rpi/pisp/data/imx477_6mm.json b/src/ipa/rpi/pisp/data/imx477_6mm.json index c8b27d6cc..4c9bdfa80 100644 --- a/src/ipa/rpi/pisp/data/imx477_6mm.json +++ b/src/ipa/rpi/pisp/data/imx477_6mm.json @@ -195,6 +195,69 @@ "transverse_neg": 0.02255 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2850.0, 0.4307, 0.3957, + 2960.0, 0.4159, 0.4313, + 3580.0, 0.3771, 0.5176, + 4559.0, 0.3031, 0.6573, + 5881.0, 0.2809, 0.6942, + 7600.0, 0.2263, 0.7762 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.02634, + "transverse_neg": 0.02255, + "ccm": + [ + 2.1643743343419066, -0.972589984871407, -0.19177768532526474, + -0.3769567095310136, 2.0993768608169443, -0.722416815431165, + -0.11786965204236007, -0.4893621633888049, 1.607231815431165 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -1240,6 +1303,6 @@ "rpi.sync": { } - } + } ] } \ No newline at end of file diff --git a/src/ipa/rpi/pisp/data/imx477_scientific.json b/src/ipa/rpi/pisp/data/imx477_scientific.json index 3cfb4d14d..855170826 100644 --- a/src/ipa/rpi/pisp/data/imx477_scientific.json +++ b/src/ipa/rpi/pisp/data/imx477_scientific.json @@ -164,6 +164,83 @@ "coarse_step": 0.1 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2000.0, 0.6331025775790707, 0.27424225990946915, + 2200.0, 0.5696117366212947, 0.3116091368689487, + 2400.0, 0.5204264653110015, 0.34892179554105873, + 2600.0, 0.48148675531667223, 0.38565229719076793, + 2800.0, 0.450085403501908, 0.42145684622485047, + 3000.0, 0.42436130159169017, 0.45611835670028816, + 3200.0, 0.40300023695527337, 0.48950766215198593, + 3400.0, 0.3850520052612984, 0.5215567075837261, + 3600.0, 0.36981508088230314, 0.5522397906415475, + 4100.0, 0.333468007836758, 0.5909770465167908, + 4600.0, 0.31196097364221376, 0.6515706327327178, + 5100.0, 0.2961860409294588, 0.7068178946570284, + 5600.0, 0.2842607232745885, 0.7564837749584288, + 6100.0, 0.2750265787051251, 0.8006183524920533, + 6600.0, 0.2677057225584924, 0.8398879225373039, + 7100.0, 0.2617955199757274, 0.8746456080032436, + 7600.0, 0.25693714288250125, 0.905569559506562, + 8100.0, 0.25287531441063316, 0.9331696750390895, + 8600.0, 0.24946601483331993, 0.9576820904825795 + ], + "sensitivity_r": 1.05, + "sensitivity_b": 1.05, + "transverse_pos": 0.0238, + "transverse_neg": 0.04429, + "coarse_step": 0.1, + "ccm": + [ + 2.003815467921944, -1.0081613204143252, 0.005840157117467748, + -0.18090523909630973, 1.597736399205449, -0.4326323675585491, + 0.05055066369087284, -0.6057020512156361, 1.5577256973300102 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -546,6 +623,6 @@ "rpi.sync": { } - } + } ] } \ No newline at end of file diff --git a/src/ipa/rpi/pisp/data/imx500.json b/src/ipa/rpi/pisp/data/imx500.json index 59a2aac56..a20546dda 100644 --- a/src/ipa/rpi/pisp/data/imx500.json +++ b/src/ipa/rpi/pisp/data/imx500.json @@ -199,6 +199,73 @@ "transverse_neg": 0.02678 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2800, + "hi": 7700 + }, + "incandescent": + { + "lo": 2800, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 7600 + } + }, + "ct_curve": + [ + 2800.0, 0.7115, 0.3579, + 2860.0, 0.6671, 0.4058, + 2880.0, 0.6641, 0.4089, + 3580.0, 0.5665, 0.5113, + 3650.0, 0.5621, 0.5159, + 4500.0, 0.4799, 0.5997, + 4570.0, 0.4752, 0.6046, + 5648.0, 0.4139, 0.6657, + 5717.0, 0.4118, 0.6678, + 7600.0, 0.3625, 0.7162 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.02822, + "transverse_neg": 0.02678, + "ccm": + [ + 1.6753287012987015, -0.4685774582560297, -0.20675124304267162, + -0.3610687012987013, 1.906408293135436, -0.5453335807050093, + -0.057295510204081634, -0.48813066790352505, 1.5454261781076069 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { diff --git a/src/ipa/rpi/pisp/data/imx708.json b/src/ipa/rpi/pisp/data/imx708.json index 6b68bbc40..7b5ee46b1 100644 --- a/src/ipa/rpi/pisp/data/imx708.json +++ b/src/ipa/rpi/pisp/data/imx708.json @@ -194,6 +194,68 @@ "transverse_neg": 0.01831 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2964.0, 0.7451, 0.3213, + 3610.0, 0.6119, 0.4443, + 4640.0, 0.5168, 0.5419, + 5910.0, 0.4436, 0.6229, + 7590.0, 0.3847, 0.6921 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.01752, + "transverse_neg": 0.01831, + "ccm": + [ + 1.5407949606299214, -0.3714970078740158, -0.16929511811023623, + -0.2801589763779528, 1.649028503937008, -0.36886236220472446, + 0.004032519685039371, -0.5251851181102363, 1.521162598425197 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { @@ -1287,6 +1349,6 @@ "rpi.sync": { } - } + } ] } diff --git a/src/ipa/rpi/pisp/data/imx708_wide.json b/src/ipa/rpi/pisp/data/imx708_wide.json index aeeb1a0dc..8760ddff6 100644 --- a/src/ipa/rpi/pisp/data/imx708_wide.json +++ b/src/ipa/rpi/pisp/data/imx708_wide.json @@ -194,6 +194,68 @@ "transverse_neg": 0.01376 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2868.0, 0.6419, 0.3613, + 3603.0, 0.5374, 0.4787, + 4620.0, 0.4482, 0.5813, + 5901.0, 0.3883, 0.6514, + 7610.0, 0.3279, 0.7232 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.01908, + "transverse_neg": 0.01376, + "ccm": + [ + 1.5820866588602653, -0.39406808743169397, -0.1880145042935207, + -0.3101711553473849, 1.756938087431694, -0.44677099921935987, + -0.018062732240437158, -0.5139293442622951, 1.5319991100702577 + ], + "enable_nn": 1 + } + }, { "rpi.agc": { diff --git a/src/ipa/rpi/pisp/data/meson.build b/src/ipa/rpi/pisp/data/meson.build index a39ce510b..e671137eb 100644 --- a/src/ipa/rpi/pisp/data/meson.build +++ b/src/ipa/rpi/pisp/data/meson.build @@ -27,5 +27,12 @@ conf_files = files([ 'uncalibrated.json', ]) +model_files = files([ + 'awb_model.tflite' +]) + install_data(conf_files, install_dir : ipa_data_dir / 'rpi' / 'pisp') + +install_data(model_files, + install_dir : ipa_data_dir / 'rpi' / 'pisp') diff --git a/src/ipa/rpi/pisp/data/ov5647.json b/src/ipa/rpi/pisp/data/ov5647.json index beb20799b..959bf589d 100644 --- a/src/ipa/rpi/pisp/data/ov5647.json +++ b/src/ipa/rpi/pisp/data/ov5647.json @@ -195,6 +195,69 @@ "transverse_neg": 0.03906 } }, + { + "disable.rpi.nn.awb": + { + "modes": + { + "auto": + { + "lo": 2500, + "hi": 7700 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8000 + } + }, + "ct_curve": + [ + 2873.0, 1.0463, 0.5142, + 2965.0, 1.0233, 0.5284, + 3606.0, 0.8947, 0.6314, + 4700.0, 0.7665, 0.7897, + 5890.0, 0.7055, 0.8933, + 7600.0, 0.6482, 1.0119 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.04072, + "transverse_neg": 0.03906, + "ccm": + [ + 2.041588151260504, -0.5494553781512606, -0.49214025210084034, + -0.5116488235294118, 1.9901442857142857, -0.47849546218487393, + -0.10519773109243696, -0.641700168067227, 1.7468953781512604 + ], + "enable_nn": 1 + } + }, { "rpi.agc": {